{
 "cells": [
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T03:16:41.356998Z",
     "start_time": "2025-01-17T03:16:41.352618Z"
    }
   },
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "from tqdm.auto import tqdm\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "print(sys.version_info)\n",
    "for module in mpl, np, pd, sklearn, torch:\n",
    "    print(module.__name__, module.__version__)\n",
    "    \n",
    "device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "print(device)\n"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sys.version_info(major=3, minor=12, micro=3, releaselevel='final', serial=0)\n",
      "matplotlib 3.10.0\n",
      "numpy 1.26.4\n",
      "pandas 2.2.3\n",
      "sklearn 1.6.0\n",
      "torch 2.5.1+cpu\n",
      "cpu\n"
     ]
    }
   ],
   "execution_count": 8
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T03:16:41.424797Z",
     "start_time": "2025-01-17T03:16:41.417251Z"
    }
   },
   "source": [
    "from sklearn.datasets import fetch_california_housing\n",
    "\n",
    "housing = fetch_california_housing()\n",
    "print(housing.DESCR)\n",
    "print(housing.data.shape)\n",
    "print(housing.target.shape)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".. _california_housing_dataset:\n",
      "\n",
      "California Housing dataset\n",
      "--------------------------\n",
      "\n",
      "**Data Set Characteristics:**\n",
      "\n",
      ":Number of Instances: 20640\n",
      "\n",
      ":Number of Attributes: 8 numeric, predictive attributes and the target\n",
      "\n",
      ":Attribute Information:\n",
      "    - MedInc        median income in block group\n",
      "    - HouseAge      median house age in block group\n",
      "    - AveRooms      average number of rooms per household\n",
      "    - AveBedrms     average number of bedrooms per household\n",
      "    - Population    block group population\n",
      "    - AveOccup      average number of household members\n",
      "    - Latitude      block group latitude\n",
      "    - Longitude     block group longitude\n",
      "\n",
      ":Missing Attribute Values: None\n",
      "\n",
      "This dataset was obtained from the StatLib repository.\n",
      "https://www.dcc.fc.up.pt/~ltorgo/Regression/cal_housing.html\n",
      "\n",
      "The target variable is the median house value for California districts,\n",
      "expressed in hundreds of thousands of dollars ($100,000).\n",
      "\n",
      "This dataset was derived from the 1990 U.S. census, using one row per census\n",
      "block group. A block group is the smallest geographical unit for which the U.S.\n",
      "Census Bureau publishes sample data (a block group typically has a population\n",
      "of 600 to 3,000 people).\n",
      "\n",
      "A household is a group of people residing within a home. Since the average\n",
      "number of rooms and bedrooms in this dataset are provided per household, these\n",
      "columns may take surprisingly large values for block groups with few households\n",
      "and many empty houses, such as vacation resorts.\n",
      "\n",
      "It can be downloaded/loaded using the\n",
      ":func:`sklearn.datasets.fetch_california_housing` function.\n",
      "\n",
      ".. rubric:: References\n",
      "\n",
      "- Pace, R. Kelley and Ronald Barry, Sparse Spatial Autoregressions,\n",
      "  Statistics and Probability Letters, 33 (1997) 291-297\n",
      "\n",
      "(20640, 8)\n",
      "(20640,)\n"
     ]
    }
   ],
   "execution_count": 9
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T03:16:41.433836Z",
     "start_time": "2025-01-17T03:16:41.430431Z"
    }
   },
   "source": [
    "# print(housing.data[0:5])\n",
    "import pprint  #打印的格式比较 好看\n",
    "\n",
    "pprint.pprint(housing.data[0:5])\n",
    "print('-'*50)\n",
    "pprint.pprint(housing.target[0:5])"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "array([[ 8.32520000e+00,  4.10000000e+01,  6.98412698e+00,\n",
      "         1.02380952e+00,  3.22000000e+02,  2.55555556e+00,\n",
      "         3.78800000e+01, -1.22230000e+02],\n",
      "       [ 8.30140000e+00,  2.10000000e+01,  6.23813708e+00,\n",
      "         9.71880492e-01,  2.40100000e+03,  2.10984183e+00,\n",
      "         3.78600000e+01, -1.22220000e+02],\n",
      "       [ 7.25740000e+00,  5.20000000e+01,  8.28813559e+00,\n",
      "         1.07344633e+00,  4.96000000e+02,  2.80225989e+00,\n",
      "         3.78500000e+01, -1.22240000e+02],\n",
      "       [ 5.64310000e+00,  5.20000000e+01,  5.81735160e+00,\n",
      "         1.07305936e+00,  5.58000000e+02,  2.54794521e+00,\n",
      "         3.78500000e+01, -1.22250000e+02],\n",
      "       [ 3.84620000e+00,  5.20000000e+01,  6.28185328e+00,\n",
      "         1.08108108e+00,  5.65000000e+02,  2.18146718e+00,\n",
      "         3.78500000e+01, -1.22250000e+02]])\n",
      "--------------------------------------------------\n",
      "array([4.526, 3.585, 3.521, 3.413, 3.422])\n"
     ]
    }
   ],
   "execution_count": 10
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T03:16:41.440017Z",
     "start_time": "2025-01-17T03:16:41.434840Z"
    }
   },
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "#拆分训练集和测试集，random_state是随机种子,同样的随机数种子，是为了得到同样的随机值\n",
    "x_train_all, x_test, y_train_all, y_test = train_test_split(\n",
    "    housing.data, housing.target, random_state = 7)\n",
    "x_train, x_valid, y_train, y_valid = train_test_split(\n",
    "    x_train_all, y_train_all, random_state = 11)\n",
    "# 训练集\n",
    "print(x_train.shape, y_train.shape)\n",
    "# 验证集\n",
    "print(x_valid.shape, y_valid.shape)\n",
    "# 测试集\n",
    "print(x_test.shape, y_test.shape)\n",
    "\n",
    "dataset_maps = {\n",
    "    \"train\": [x_train, y_train],\n",
    "    \"valid\": [x_valid, y_valid],\n",
    "    \"test\": [x_test, y_test],\n",
    "}\n"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11610, 8) (11610,)\n",
      "(3870, 8) (3870,)\n",
      "(5160, 8) (5160,)\n"
     ]
    }
   ],
   "execution_count": 11
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T03:16:41.459106Z",
     "start_time": "2025-01-17T03:16:41.454540Z"
    }
   },
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "\n",
    "scaler = StandardScaler()\n",
    "scaler.fit(x_train)"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "StandardScaler()"
      ],
      "text/html": [
       "<style>#sk-container-id-2 {\n",
       "  /* Definition of color scheme common for light and dark mode */\n",
       "  --sklearn-color-text: #000;\n",
       "  --sklearn-color-text-muted: #666;\n",
       "  --sklearn-color-line: gray;\n",
       "  /* Definition of color scheme for unfitted estimators */\n",
       "  --sklearn-color-unfitted-level-0: #fff5e6;\n",
       "  --sklearn-color-unfitted-level-1: #f6e4d2;\n",
       "  --sklearn-color-unfitted-level-2: #ffe0b3;\n",
       "  --sklearn-color-unfitted-level-3: chocolate;\n",
       "  /* Definition of color scheme for fitted estimators */\n",
       "  --sklearn-color-fitted-level-0: #f0f8ff;\n",
       "  --sklearn-color-fitted-level-1: #d4ebff;\n",
       "  --sklearn-color-fitted-level-2: #b3dbfd;\n",
       "  --sklearn-color-fitted-level-3: cornflowerblue;\n",
       "\n",
       "  /* Specific color for light theme */\n",
       "  --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, white)));\n",
       "  --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-icon: #696969;\n",
       "\n",
       "  @media (prefers-color-scheme: dark) {\n",
       "    /* Redefinition of color scheme for dark theme */\n",
       "    --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, #111)));\n",
       "    --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-icon: #878787;\n",
       "  }\n",
       "}\n",
       "\n",
       "#sk-container-id-2 {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 pre {\n",
       "  padding: 0;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 input.sk-hidden--visually {\n",
       "  border: 0;\n",
       "  clip: rect(1px 1px 1px 1px);\n",
       "  clip: rect(1px, 1px, 1px, 1px);\n",
       "  height: 1px;\n",
       "  margin: -1px;\n",
       "  overflow: hidden;\n",
       "  padding: 0;\n",
       "  position: absolute;\n",
       "  width: 1px;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-dashed-wrapped {\n",
       "  border: 1px dashed var(--sklearn-color-line);\n",
       "  margin: 0 0.4em 0.5em 0.4em;\n",
       "  box-sizing: border-box;\n",
       "  padding-bottom: 0.4em;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-container {\n",
       "  /* jupyter's `normalize.less` sets `[hidden] { display: none; }`\n",
       "     but bootstrap.min.css set `[hidden] { display: none !important; }`\n",
       "     so we also need the `!important` here to be able to override the\n",
       "     default hidden behavior on the sphinx rendered scikit-learn.org.\n",
       "     See: https://github.com/scikit-learn/scikit-learn/issues/21755 */\n",
       "  display: inline-block !important;\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-text-repr-fallback {\n",
       "  display: none;\n",
       "}\n",
       "\n",
       "div.sk-parallel-item,\n",
       "div.sk-serial,\n",
       "div.sk-item {\n",
       "  /* draw centered vertical line to link estimators */\n",
       "  background-image: linear-gradient(var(--sklearn-color-text-on-default-background), var(--sklearn-color-text-on-default-background));\n",
       "  background-size: 2px 100%;\n",
       "  background-repeat: no-repeat;\n",
       "  background-position: center center;\n",
       "}\n",
       "\n",
       "/* Parallel-specific style estimator block */\n",
       "\n",
       "#sk-container-id-2 div.sk-parallel-item::after {\n",
       "  content: \"\";\n",
       "  width: 100%;\n",
       "  border-bottom: 2px solid var(--sklearn-color-text-on-default-background);\n",
       "  flex-grow: 1;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-parallel {\n",
       "  display: flex;\n",
       "  align-items: stretch;\n",
       "  justify-content: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-parallel-item {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-parallel-item:first-child::after {\n",
       "  align-self: flex-end;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-parallel-item:last-child::after {\n",
       "  align-self: flex-start;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-parallel-item:only-child::after {\n",
       "  width: 0;\n",
       "}\n",
       "\n",
       "/* Serial-specific style estimator block */\n",
       "\n",
       "#sk-container-id-2 div.sk-serial {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "  align-items: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  padding-right: 1em;\n",
       "  padding-left: 1em;\n",
       "}\n",
       "\n",
       "\n",
       "/* Toggleable style: style used for estimator/Pipeline/ColumnTransformer box that is\n",
       "clickable and can be expanded/collapsed.\n",
       "- Pipeline and ColumnTransformer use this feature and define the default style\n",
       "- Estimators will overwrite some part of the style using the `sk-estimator` class\n",
       "*/\n",
       "\n",
       "/* Pipeline and ColumnTransformer style (default) */\n",
       "\n",
       "#sk-container-id-2 div.sk-toggleable {\n",
       "  /* Default theme specific background. It is overwritten whether we have a\n",
       "  specific estimator or a Pipeline/ColumnTransformer */\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "/* Toggleable label */\n",
       "#sk-container-id-2 label.sk-toggleable__label {\n",
       "  cursor: pointer;\n",
       "  display: flex;\n",
       "  width: 100%;\n",
       "  margin-bottom: 0;\n",
       "  padding: 0.5em;\n",
       "  box-sizing: border-box;\n",
       "  text-align: center;\n",
       "  align-items: start;\n",
       "  justify-content: space-between;\n",
       "  gap: 0.5em;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 label.sk-toggleable__label .caption {\n",
       "  font-size: 0.6rem;\n",
       "  font-weight: lighter;\n",
       "  color: var(--sklearn-color-text-muted);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 label.sk-toggleable__label-arrow:before {\n",
       "  /* Arrow on the left of the label */\n",
       "  content: \"▸\";\n",
       "  float: left;\n",
       "  margin-right: 0.25em;\n",
       "  color: var(--sklearn-color-icon);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "/* Toggleable content - dropdown */\n",
       "\n",
       "#sk-container-id-2 div.sk-toggleable__content {\n",
       "  max-height: 0;\n",
       "  max-width: 0;\n",
       "  overflow: hidden;\n",
       "  text-align: left;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-toggleable__content.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-toggleable__content pre {\n",
       "  margin: 0.2em;\n",
       "  border-radius: 0.25em;\n",
       "  color: var(--sklearn-color-text);\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-toggleable__content.fitted pre {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {\n",
       "  /* Expand drop-down */\n",
       "  max-height: 200px;\n",
       "  max-width: 100%;\n",
       "  overflow: auto;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {\n",
       "  content: \"▾\";\n",
       "}\n",
       "\n",
       "/* Pipeline/ColumnTransformer-specific style */\n",
       "\n",
       "#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-label.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator-specific style */\n",
       "\n",
       "/* Colorize estimator box */\n",
       "#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-estimator.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-label label.sk-toggleable__label,\n",
       "#sk-container-id-2 div.sk-label label {\n",
       "  /* The background is the default theme color */\n",
       "  color: var(--sklearn-color-text-on-default-background);\n",
       "}\n",
       "\n",
       "/* On hover, darken the color of the background */\n",
       "#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "/* Label box, darken color on hover, fitted */\n",
       "#sk-container-id-2 div.sk-label.fitted:hover label.sk-toggleable__label.fitted {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator label */\n",
       "\n",
       "#sk-container-id-2 div.sk-label label {\n",
       "  font-family: monospace;\n",
       "  font-weight: bold;\n",
       "  display: inline-block;\n",
       "  line-height: 1.2em;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-label-container {\n",
       "  text-align: center;\n",
       "}\n",
       "\n",
       "/* Estimator-specific */\n",
       "#sk-container-id-2 div.sk-estimator {\n",
       "  font-family: monospace;\n",
       "  border: 1px dotted var(--sklearn-color-border-box);\n",
       "  border-radius: 0.25em;\n",
       "  box-sizing: border-box;\n",
       "  margin-bottom: 0.5em;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-estimator.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "/* on hover */\n",
       "#sk-container-id-2 div.sk-estimator:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-estimator.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Specification for estimator info (e.g. \"i\" and \"?\") */\n",
       "\n",
       "/* Common style for \"i\" and \"?\" */\n",
       "\n",
       ".sk-estimator-doc-link,\n",
       "a:link.sk-estimator-doc-link,\n",
       "a:visited.sk-estimator-doc-link {\n",
       "  float: right;\n",
       "  font-size: smaller;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1em;\n",
       "  height: 1em;\n",
       "  width: 1em;\n",
       "  text-decoration: none !important;\n",
       "  margin-left: 0.5em;\n",
       "  text-align: center;\n",
       "  /* unfitted */\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted,\n",
       "a:link.sk-estimator-doc-link.fitted,\n",
       "a:visited.sk-estimator-doc-link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "div.sk-estimator:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "div.sk-estimator.fitted:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "/* Span, style for the box shown on hovering the info icon */\n",
       ".sk-estimator-doc-link span {\n",
       "  display: none;\n",
       "  z-index: 9999;\n",
       "  position: relative;\n",
       "  font-weight: normal;\n",
       "  right: .2ex;\n",
       "  padding: .5ex;\n",
       "  margin: .5ex;\n",
       "  width: min-content;\n",
       "  min-width: 20ex;\n",
       "  max-width: 50ex;\n",
       "  color: var(--sklearn-color-text);\n",
       "  box-shadow: 2pt 2pt 4pt #999;\n",
       "  /* unfitted */\n",
       "  background: var(--sklearn-color-unfitted-level-0);\n",
       "  border: .5pt solid var(--sklearn-color-unfitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted span {\n",
       "  /* fitted */\n",
       "  background: var(--sklearn-color-fitted-level-0);\n",
       "  border: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link:hover span {\n",
       "  display: block;\n",
       "}\n",
       "\n",
       "/* \"?\"-specific style due to the `<a>` HTML tag */\n",
       "\n",
       "#sk-container-id-2 a.estimator_doc_link {\n",
       "  float: right;\n",
       "  font-size: 1rem;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1rem;\n",
       "  height: 1rem;\n",
       "  width: 1rem;\n",
       "  text-decoration: none;\n",
       "  /* unfitted */\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 a.estimator_doc_link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "#sk-container-id-2 a.estimator_doc_link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 a.estimator_doc_link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>StandardScaler()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator fitted sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" checked><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label fitted sk-toggleable__label-arrow\"><div><div>StandardScaler</div></div><div><a class=\"sk-estimator-doc-link fitted\" rel=\"noreferrer\" target=\"_blank\" href=\"https://scikit-learn.org/1.6/modules/generated/sklearn.preprocessing.StandardScaler.html\">?<span>Documentation for StandardScaler</span></a><span class=\"sk-estimator-doc-link fitted\">i<span>Fitted</span></span></div></label><div class=\"sk-toggleable__content fitted\"><pre>StandardScaler()</pre></div> </div></div></div></div>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 12
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 构建数据集"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T03:16:41.472012Z",
     "start_time": "2025-01-17T03:16:41.467125Z"
    }
   },
   "source": [
    "from torch.utils.data import Dataset\n",
    "\n",
    "class HousingDataset(Dataset):\n",
    "    def __init__(self, mode='train'):\n",
    "        self.x, self.y = dataset_maps[mode]\n",
    "        self.x = torch.from_numpy(scaler.transform(self.x)).float()\n",
    "        self.y = torch.from_numpy(self.y).float().reshape(-1, 1)\n",
    "            \n",
    "    def __len__(self):\n",
    "        return len(self.x)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        return self.x[idx], self.y[idx]\n",
    "    \n",
    "    \n",
    "train_ds = HousingDataset(\"train\")\n",
    "valid_ds = HousingDataset(\"valid\")\n",
    "test_ds = HousingDataset(\"test\")"
   ],
   "outputs": [],
   "execution_count": 13
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T03:16:41.477209Z",
     "start_time": "2025-01-17T03:16:41.472012Z"
    }
   },
   "source": [
    "train_ds[1]"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([-0.2981,  0.3523, -0.1092, -0.2506, -0.0341, -0.0060,  1.0806, -1.0611]),\n",
       " tensor([1.5140]))"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 14
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T03:16:41.479844Z",
     "start_time": "2025-01-17T03:16:41.477209Z"
    }
   },
   "source": [
    "from torch.utils.data import DataLoader\n",
    "\n",
    "\n",
    "batch_size = 8\n",
    "train_loader = DataLoader(train_ds, batch_size=batch_size, shuffle=False)\n",
    "val_loader = DataLoader(valid_ds, batch_size=batch_size, shuffle=False)\n",
    "test_loader = DataLoader(test_ds, batch_size=batch_size, shuffle=False)"
   ],
   "outputs": [],
   "execution_count": 15
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 定义模型\n",
    "\n",
    "多输出模型常见于多任务学习（Multi-Task Learning）中，一个任务会有一个输出层，计算一个loss，最后多个任务的loss联合训练模型。除此之外，如果想拿到模型的中间输出，也可以使用多输出。\n",
    "\n",
    "例如，这里想拿到 deep_output，我们构建多输出模型"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T03:16:41.484941Z",
     "start_time": "2025-01-17T03:16:41.481849Z"
    }
   },
   "source": [
    "#回归模型我们只需要1个数\n",
    "\n",
    "class WideDeep(nn.Module):\n",
    "    def __init__(self, input_dim=8):\n",
    "        super().__init__()\n",
    "        self.deep = nn.Sequential(\n",
    "            nn.Linear(input_dim, 30),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(30, 30),\n",
    "            nn.ReLU()\n",
    "            )\n",
    "        # pytorch 需要自行计算输出输出维度\n",
    "        self.output_layer = nn.Linear(30 + input_dim, 1)\n",
    "        \n",
    "        # 初始化权重\n",
    "        self.init_weights()\n",
    "        \n",
    "    def init_weights(self):\n",
    "        \"\"\"使用 xavier 均匀分布来初始化全连接层的权重 W\"\"\"\n",
    "        for m in self.modules():\n",
    "            if isinstance(m, nn.Linear):\n",
    "                nn.init.xavier_uniform_(m.weight)\n",
    "                nn.init.zeros_(m.bias)\n",
    "        \n",
    "    def forward(self, x, return_deep_output=False):\n",
    "        # x.shape [batch size, 8]\n",
    "        deep_output = self.deep(x)\n",
    "        # concat [batch size, 30] with [batch size 8]\n",
    "        concat = torch.cat([x, deep_output], dim=-1)\n",
    "        logits = self.output_layer(concat)\n",
    "        # logits.shape [batch size, 1]\n",
    "        return (logits, deep_output) if return_deep_output else logits #如果return_deep_output为True，返回logits和deep_output，否则只返回logits"
   ],
   "outputs": [],
   "execution_count": 16
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T03:16:41.488041Z",
     "start_time": "2025-01-17T03:16:41.484941Z"
    }
   },
   "source": [
    "class EarlyStopCallback:\n",
    "    def __init__(self, patience=5, min_delta=0.01):\n",
    "        \"\"\"\n",
    "\n",
    "        Args:\n",
    "            patience (int, optional): Number of epochs with no improvement after which training will be stopped.. Defaults to 5.\n",
    "            min_delta (float, optional): Minimum change in the monitored quantity to qualify as an improvement, i.e. an absolute \n",
    "                change of less than min_delta, will count as no improvement. Defaults to 0.01.\n",
    "        \"\"\"\n",
    "        self.patience = patience\n",
    "        self.min_delta = min_delta\n",
    "        self.best_metric = -1\n",
    "        self.counter = 0\n",
    "        \n",
    "    def __call__(self, metric):\n",
    "        if metric >= self.best_metric + self.min_delta:\n",
    "            # update best metric\n",
    "            self.best_metric = metric\n",
    "            # reset counter \n",
    "            self.counter = 0\n",
    "        else: \n",
    "            self.counter += 1\n",
    "            \n",
    "    @property\n",
    "    def early_stop(self):\n",
    "        return self.counter >= self.patience\n"
   ],
   "outputs": [],
   "execution_count": 17
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T03:16:41.491333Z",
     "start_time": "2025-01-17T03:16:41.488041Z"
    }
   },
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "@torch.no_grad()\n",
    "def evaluating(model, dataloader, loss_fct):\n",
    "    loss_list = []\n",
    "    for datas, labels in dataloader:\n",
    "        datas = datas.to(device)\n",
    "        labels = labels.to(device)\n",
    "        # 前向计算\n",
    "        logits = model(datas)\n",
    "        loss = loss_fct(logits, labels)         # 验证集损失\n",
    "        loss_list.append(loss.item())\n",
    "        \n",
    "    return np.mean(loss_list)\n"
   ],
   "outputs": [],
   "execution_count": 18
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": "## 训练(不重要，不用花时间看）"
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T03:16:55.321318Z",
     "start_time": "2025-01-17T03:16:41.495838Z"
    }
   },
   "source": [
    "# 训练\n",
    "def training(\n",
    "    model, \n",
    "    train_loader, \n",
    "    val_loader, \n",
    "    epoch, \n",
    "    loss_fct, \n",
    "    optimizer, \n",
    "    tensorboard_callback=None,\n",
    "    save_ckpt_callback=None,\n",
    "    early_stop_callback=None,\n",
    "    eval_step=500,\n",
    "    ):\n",
    "    record_dict = {\n",
    "        \"train\": [],\n",
    "        \"val\": []\n",
    "    }\n",
    "    \n",
    "    global_step = 0\n",
    "    model.train()\n",
    "    with tqdm(total=epoch * len(train_loader)) as pbar:\n",
    "        for epoch_id in range(epoch):\n",
    "            # training\n",
    "            for datas, labels in train_loader:\n",
    "                datas = datas.to(device)\n",
    "                labels = labels.to(device)\n",
    "                # 梯度清空\n",
    "                optimizer.zero_grad()\n",
    "                # 模型前向计算\n",
    "                logits, deep_output = model(datas,return_deep_output=True)\n",
    "                deep_output= deep_output.mean(axis=1).reshape(-1, 1) # # 平均池化,尺寸为[batch size, 30]，求平均就变为[batch size],reshape成[batch size, 1]\n",
    "                logits=logits+deep_output  # 尺寸一致，相加，求损失\n",
    "                # 计算损失\n",
    "                loss = loss_fct(logits, labels)\n",
    "                # 梯度回传\n",
    "                loss.backward()\n",
    "                # 调整优化器，包括学习率的变动等\n",
    "                optimizer.step()\n",
    " \n",
    "                loss = loss.cpu().item()\n",
    "                # record\n",
    "                \n",
    "                record_dict[\"train\"].append({\n",
    "                    \"loss\": loss, \"step\": global_step\n",
    "                })\n",
    "                \n",
    "                # evaluating\n",
    "                if global_step % eval_step == 0:\n",
    "                    model.eval()\n",
    "                    val_loss = evaluating(model, val_loader, loss_fct)\n",
    "                    record_dict[\"val\"].append({\n",
    "                        \"loss\": val_loss, \"step\": global_step\n",
    "                    })\n",
    "                    model.train()\n",
    "\n",
    "                    # 早停 Early Stop\n",
    "                    if early_stop_callback is not None:\n",
    "                        early_stop_callback(-val_loss)\n",
    "                        if early_stop_callback.early_stop:\n",
    "                            print(f\"Early stop at epoch {epoch_id} / global_step {global_step}\")\n",
    "                            return record_dict\n",
    "                    \n",
    "                # udate step\n",
    "                global_step += 1\n",
    "                pbar.update(1)\n",
    "                pbar.set_postfix({\"epoch\": epoch_id})\n",
    "        \n",
    "    return record_dict\n",
    "        \n",
    "\n",
    "epoch = 10\n",
    "\n",
    "model = WideDeep()\n",
    "\n",
    "# 1. 定义损失函数 采用MSE损失\n",
    "loss_fct = nn.MSELoss()\n",
    "# 2. 定义优化器 采用SGD\n",
    "# Optimizers specified in the torch.optim package\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=0.001, momentum=0.0)\n",
    "\n",
    "# 3. early stop\n",
    "early_stop_callback = EarlyStopCallback(patience=10, min_delta=1e-3)\n",
    "\n",
    "model = model.to(device)\n",
    "record = training(\n",
    "    model, \n",
    "    train_loader, \n",
    "    val_loader, \n",
    "    epoch, \n",
    "    loss_fct, \n",
    "    optimizer, \n",
    "    early_stop_callback=early_stop_callback,\n",
    "    eval_step=len(train_loader)\n",
    "    )"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "  0%|          | 0/14520 [00:00<?, ?it/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "5e3636436b0a45f3bca2297c5adbf24c"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The Jupyter server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--ServerApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "ServerApp.iopub_msg_rate_limit=2000.0 (msgs/sec)\n",
      "ServerApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The Jupyter server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--ServerApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "ServerApp.iopub_msg_rate_limit=2000.0 (msgs/sec)\n",
      "ServerApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The Jupyter server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--ServerApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "ServerApp.iopub_msg_rate_limit=2000.0 (msgs/sec)\n",
      "ServerApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    }
   ],
   "execution_count": 19
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T03:16:55.383962Z",
     "start_time": "2025-01-17T03:16:55.321318Z"
    }
   },
   "source": [
    "#画线要注意的是损失是不一定在零到1之间的\n",
    "def plot_learning_curves(record_dict, sample_step=500):\n",
    "    # build DataFrame\n",
    "    train_df = pd.DataFrame(record_dict[\"train\"]).set_index(\"step\").iloc[::sample_step]\n",
    "    val_df = pd.DataFrame(record_dict[\"val\"]).set_index(\"step\")\n",
    "\n",
    "    # plot\n",
    "    for idx, item in enumerate(train_df.columns):\n",
    "        plt.plot(train_df.index, train_df[item], label=f\"train_{item}\")\n",
    "        plt.plot(val_df.index, val_df[item], label=f\"val_{item}\")\n",
    "        plt.grid()\n",
    "        plt.legend()\n",
    "        # plt.xticks(range(0, train_df.index[-1], 10*sample_step), range(0, train_df.index[-1], 10*sample_step))\n",
    "        plt.xlabel(\"step\")\n",
    "\n",
    "        plt.show()\n",
    "\n",
    "plot_learning_curves(record, sample_step=500)  #横坐标是 steps"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ],
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGwCAYAAAAJ/wd3AAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAV69JREFUeJzt3QWYW1XaB/B/Mp50ZipT79Td3SlSxW2Rtvgu2i4tsCywu0CLU6xQHD5kgQpWYIuWuht1L3X3js9kJvd73pPcNDMdycwkN3L/v+dJo03uySS5733Pe86xaJqmgYiIiMggVqNeiIiIiEgw+CAiIiJDMfggIiIiQzH4ICIiIkMx+CAiIiJDMfggIiIiQzH4ICIiIkNFI8Q4nU4cPHgQiYmJsFgswd4cIiIi8oFMG5aeno569erBarWGV/AhgUdqamqwN4OIiIgqYN++fWjQoEF4BR+S8dA3Pikpya/P7XA48Ntvv2HIkCGIiYmBWbDdbLdZmLXtbDfbHQrS0tJU8kDfj4dV8KF3tUjgEYjgw2azqecNpT9YoLHdbLdZmLXtbDfbHUp8KZlgwSkREREZisEHERERGYrBBxERERkq5Go+iIgoMhUUFKh6BX+R54qOjkZOTo56brNwBLHdsbGxZQ6j9QWDDyIiCvj8D4cPH8bp06f9/rx16tRRoyPNNC+UFsR2S+DRpEkTFYRUBoMPIiIKKD3wqFWrlhql4a8dpkxKmZGRgSpVqvjlaDxcOIPUbn0S0EOHDqFhw4aV+jsy+CAiooCRbgE98KhRo4bfd4Z5eXmIj483XfCRF6R216xZUwUg+fn5lRrma56/FhERGU6v8ZCMB4W/WHd3S2VrTRh8EBFRwJmpJiOSWfz0d2TwQURERIZi8EFERESGYvBBREQUYI0bN8bEiRP98lxz585FtWrV/D502UimGe2SX+DE4bQcHM8J9pYQEVE4uOCCC9C5c2e/BA0rVqyA3W73y3ZFAtNkPv48lonzXpqP19ZHBXtTiIgoQib7kiGnvg5R5YgfEwYftlhX0JHrDPaWEBGZm+y0s/Ly/XLKziso1+PltX1x2223Yd68eXj99dfVCA85ffLJJ+r8559/Rrdu3RAXF4eFCxfizz//xJVXXonatWurib969OiB33//vdRuF4vFgg8//BBXX321CkpatGiBH374ocLv6TfffIN27dqpbZLXeuWVVwrd//bbb6vXkLlBZDv/8pe/eO77+uuv0aFDByQkJKi5WAYNGoTMzEwEkmm6XarEuZrqcFpQ4NRQ8alRiIioMrIdBWj7xK9Bee1NTw2FLbbsXZ8EHdu2bUP79u3x1FNPqds2btyozh999FG8/PLLaNq0qaq9kGnOL7nkEjz77LNq5//f//4Xl19+ObZu3apmAi3J+PHjMWHCBLz00kuYNGkSRo4ciT179qB69erlatOqVatw/fXXY9y4cbjhhhuwePFi3HfffSqQkCBq5cqVuP/++/HZZ5+hb9++OHnyJBYsWKD+r8xWOnz4cLUdEgilp6er+3wN0irKNMGHLe5sd0tWXgHi44K6OUREFMKSk5PVhFqSlZB1VMSWLVvUuQQjgwcP9jxWgoVOnTp5rj/99NOYPn26ymSMHj26xNe47bbb1I5fPPfcc3jjjTewfPlyDBs2rFzb+uqrr2LgwIF4/PHH1fWWLVti06ZNKqiR19i7d6+qN7nsssuQmJiIRo0aoUuXLp7gQ7qOrrnmGnW7kCxIoJkm+IiNsiLaakG+05XuK19cSURE/pIQE6UyEP6YZjw9LR2JSYk+TzMur11Z3bt3L3Rd1lmRrMOPP/7o2ZlnZ2ernX5pOnbs6LkswUFSUhKOHj1a7u3ZvHmz6vbx1q9fP9XNIzORSqAkgYVkaiSwkZPe3SNBkwQuEnAMHToUQ4YMUV0yktEJJNPUfEj/ml73IZkPIiIK5u9xtF9OCbFR5Xq8P2boLDpq5R//+IfKdEj2Qros1qxZo3bmsv5KaWKKrI0i2yYBlb9JtuOPP/7AlClTULduXTzxxBMq6JChulFRUZg5c6aqY2nbtq3q/mnVqhV27dqFQDJN8CEYfBARka+k28WXNUwWLVqkujckmyBBh3TT7N69G0Zp06aN2oai2yTdLxJciOjoaFVIKrUd69atU9s3e/ZsT9AjmRKpQVm9erVqtwRTgWSabhfhKjLKRUaub0OjiIjIvGTUyLJly9SOWkaxlJSVkFEk3377rSoylR251F4EIoNRkoceekiNsJFaEyk4XbJkCd588001wkXMmDEDO3fuxIABA1R3yk8//aS2TzIc0r5Zs2ap7hZZeViuHzt2TAU0gWSqzIfdXXTKzAcREZVFulMkcyDdETJPR0k1HFLwKTt1GUkiAYjUTnTt2tWw7ezatSu+/PJLTJ06VY3OkW4VKYqVbIyoWrWqCo4uuugiFVS8++67qgtGhuZKncn8+fPVaB3JlPznP/9Rw3QvvvjigG6zyTIfDD6IiMg3sjOWLII3fYdeNEOid2HoRo0aVeh60W4YrZihrL5Oly4zr546dUoFDrprr71WnYrTv39/NSV7cSQY+eWXX2A0c2U+3GO7ZbQLERERBYfVjJmPTGY+iIgoRN1zzz2qxqS4k9wXCaJNWfORy+CDiIhC01NPPaXqTYrj3dUSzkwVfOhT6rLmg4iIQlWtWrXUqSRGjqQJFFN2u7Dmg4iIKHhMGXxkMPNBREQUNKYKPux65oOTjBEREQWNqYIP1nwQEREFn6mCD85wSkREFIbBh0zDKtPH1qtXT81h/913350za5tM7Sor5yUkJKiFbLZv345Qynxwng8iIgo0mflUlrX3haWY/WkkK3fwkZmZqZbifeutt4q9X1bMe+ONN9Tc8bJAjSw9LPPc5+TkIGRqPjjahYiIKHzm+ZDFZkpacEayHhLlycI0V155pbrtv//9L2rXrq0iuhtvvPGc/5Obm6tOurS0NHXucDjUyZ9ira659DNzC/z+3KFMb6uZ2izYbnO128xtD+V2yzbJvkHmpvD3/BT6+ij684ei8myb08f3KJjtlteT15W/qyy65608nz+/TjK2a9cuHD58WHW16JKTk9GrVy+1OE9xwcfzzz+P8ePHn3P7b7/9BpvN5s/NwwmVfIlGenauWlLYbGbOnAkzYrvNx6xtD8V2R0dHo06dOsjIyEBeXp7rRtl55mf77TXST2aVY4MSpI+jzId98sknePHFF7Fx40ZYrWc7CUaMGIHq1aurZez//e9/Y+XKlcjKylKL0EnJgSz65r2jlqy/flBdluzsbM9j5XUfe+wxrFixQpUwXHHFFXjmmWfUFOti4cKFePLJJ7Flyxb1Hrdu3RoffPABGjZsiPXr1+Nf//oX1qxZo7pzmjZtitdeew1dunRBZcnfULZTSjDy8wv3Isj7EJTgQwIPIZkOb3Jdv68oeXMffPBBz3V541NTUzFkyBC/TyN75HQmnlq9CHlOC4YNuxhWa9kfwEgg0aj8KA0ePBgxMTEwC7bbXO02c9tDud2y8923b5/aacbHx7tuzMuE9YU2Qdke56P7gVh7mY+7+eab8cgjj2DVqlUYOHCguu3kyZOYNWsWZsyYoa5L/eMLL7yAuLg4fPbZZxg+fDg2b96sAgAhQYu02dd9WUJCgnqslDdcd9116N27typfOHr0KO666y4V7Hz88cfq7z1y5Ej87W9/w9SpU1VAsHz5cvV/5XTvvfeic+fOeO+991R2QoKQqlWr+mWfKn9P2c4BAwac/Xu6+RpkhcT06vJHk1NR8gXy95co2X72jXLAiioxQW++oQLxnoYDttt8zNr2UGx3QUGBOvqWHbEng+CVSTCa2gYfXr9GjRqqxEB27hLUiW+//RYpKSkqGJHn8c4kSFZCygskMBk9erTndr3tvm6b1WpVryk7eQlopG5SvPnmmyrYkbpKCShkR3/ZZZehRYsW6v527dp5nmfv3r14+OGH0bZtW3W9VatWPr8/vmyjtKm4z1p5Pnt+3ftKak0cOXJEjXbRyXWJwoItLtoKCzRosKiJxqrEmSv4ICIKCTE24F8HK/000q2Rlp6OpMREn3fw6rV9JNmFO++8E2+//bY6SP7iiy9U+YC8lnQjjRs3Dj/++CMOHTqkuiCkO0J2/JW1efNmNbBDDzxEv379VHu3bt2K/v37q+4fCY4kMJJSh+uvv96z35XeBMmKSPAi90kWpVmzZgglfg0/mzRpogIQSUvpJDqTtFGfPn0QbBKtxbvrYzjclogoSKTmQro+/HGSYKI8j/eh3kMnmQYprpQAQ7qOFixYoAISIavOTp8+Hc8995y6Xbo2OnTocLauJcDeeustLFq0CH379sW0adNUzcnSpUvVfRIUSc3IpZdeitmzZ6sMiGxrWAcfEu3JmywnvchULku0Jzv3sWPHqvTTDz/8oIpebrnlFjUnyFVXXYVQ4B5ti0xOsU5ERKWQmoZrrrlGZTymTJmiui+6du2q7pMd/2233Yarr75aBR1y4L17926/vG6bNm2wdu1aVfuhk9eTjIt3F4p0+0jd5OLFi9G+fXtMnjzZc58EIw888IAavCFtkFqRsA4+pLJXGqz3dUl6Ry5Lla/45z//ib///e+qOKZHjx4qWPnll1/OKUwJljh3iznLKRERlUUyHZL5+OijjzxZDyG1FlIDIgffEihIN4i/hr2OHDlS7TNvvfVWbNiwAXPmzFH7VSmClQEcctAvo0RlFOmePXtUgCGTeUrQIl0/UnMyd+5cdZ8ELTJiRu4LJeUuepBhRPoY4+JI9uOpp55Sp1DknmEdmZxojIiIynDRRRepobVSayEBhu7VV1/FHXfcobo9pAhVRsaUZ7RHaWSaiV9//RVjxoxRB/Fy/dprr1Wvqd8vwYbUcpw4cULVeowaNQp33323qj2R26TXQeotZdsk81HclBbBZLqKSz34yMpl5oOIiEonXR0HDx4sdup0qafwJgGAt/J0w2hFDuqlK6fo8+sk+/H555+robNFC21jY2NVF1GoM9XCcoVnOWXmg4iIKBhMF3yw24WIiIwkBasyyVpxp3Ze83OYiXm7XVhwSkREBpCp0WWZkeLEhNikcEYxbfDBbhciIjJCYmKiOpGZu1041JaIyHChuuoslU9po13Lw4SZDxacEhEZRUZf6CNGatasqa7LlAz+CmhkRlFZB8Xn6dUjgDNI7ZbA49ixY561XSrDhMGH65yZDyKiwJOdoyy9IeufFDdktbI7Q5lUS1ZZ9VdAEw60ILZbXq9BgwZqcbvKMF/w4Q4SOdqFiMgYku2QZeZlAixZ5dZfZGn5+fPnq+XdzVS46Qhiu+X1Kht4mDL44NouRETGK2kZ9sqQnaAENDIVuZmCj6gIaLd5OsnOGe3CbhciIqJgMF/w4Z7hNIvdLkREREFh4hlOmfkgIiIKBtMGH1ms+SAiIgoK8wYfjgI4nf6ZLIWIiIh8Z77gw91imaQtJ59dL0REREYzXfARY5UhX67LHPFCRERkPNMFHxJ42NyTfXCuDyIiIuOZLvgQ9ljX3Gqc5ZSIiMh4pgw+9MwH13chIiIynimDD7t7yAu7XYiIiIxnyuDD5u52YeaDiIjIeCYNPpj5ICIiChZTBh921nwQEREFjam7XTjahYiIyHgmDT7Y7UJERBQspu524QynRERExjP5PB/MfBARERnNlMGHPU6v+WDmg4iIyGjmznyw5oOIiMhwpg4+mPkgIiIynqm7XVjzQUREZDyTd7sw80FERGQ0UwcfGaz5ICIiMpwpgw87F5YjIiIKGpMXnOZD07Rgbw4REZGpmDL4sMe5gg+JO3IczmBvDhERkamYMviIj46CxeK6zMXliIiIjGXK4MNqtcAWwxEvREREwWDK4EPYPFOsM/NBRERkJNMGH/rKtpxojIiIyFimDT5s7uG2Gex2ISIiMpRpgw99xAsXlyMiIjKW1eyZDy4uR0REZCzTBh9VuLgcERFRUJg2+PDMcsqaDyIiIkOZNviwM/NBREQUFKYNPpj5ICIiCg7TBh/MfBAREQWH1eyZjwwOtSUiIjKUaYMPu3uobRaH2hIRERnKtMGHzT3JWCYzH0REROEdfBQUFODxxx9HkyZNkJCQgGbNmuHpp5+GpmkIzZoPZj6IiIiM5NoD+9GLL76Id955B59++inatWuHlStX4vbbb0dycjLuv/9+hAq7Z4ZTZj6IiIjCOvhYvHgxrrzySlx66aXqeuPGjTFlyhQsX74coVhwmsWhtkREROEdfPTt2xfvv/8+tm3bhpYtW2Lt2rVYuHAhXn311WIfn5ubq066tLQ0de5wONTJn/Tnk/O4KM2T+fD364Qa73abCdttrnabue1sN9sdCsqzPRbNz8UYTqcT//rXvzBhwgRERUWpGpBnn30Wjz32WLGPHzduHMaPH3/O7ZMnT4bNZkOgnMkDnlgVDSs0vNq7ABZLwF6KiIgo4mVlZWHEiBE4c+YMkpKSjA0+pk6diocffhgvvfSSqvlYs2YNxo4dqzIft956q0+Zj9TUVBw/frzMja9IVDZz5kwMHjwYuU4LujwzW92+/omBiI9xdcNEIu92x8TEwCzYbnO128xtZ7vZ7lAg+++UlBSfgg+/d7tI4PHoo4/ixhtvVNc7dOiAPXv24Pnnny82+IiLi1OnouQNDdSbKs8bF3W26XlOCxJD6A8YKIF8T0MZ220+Zm07220uMSHW7vJsizUQaRertfDTSveLdMeEkiirBQnubAeH2xIRERnH75mPyy+/XNV4NGzYUHW7rF69WnW53HHHHQg1MtdHtqOAw22JiIjCOfiYNGmSmmTsvvvuw9GjR1GvXj3cfffdeOKJJxBq7HFROJ7BlW2JiIjCOvhITEzExIkT1SnU2TzruzDzQUREZBTTru0i7O6Jxpj5ICIiMo6pgw+bZ30XZj6IiIiMYurg42zmg8EHERGRUUwdfOg1H5kcaktERGQYUwcfMtpFZDHzQUREZBiTBx/MfBARERnN3MGHu+aDBadERETGMXXw4an54FBbIiIiw5g6+PDUfDDzQUREZBhTBx965iODBadERESGMXXwcTbzwW4XIiIio5g6+Dhb88HMBxERkVFMHXzYPQvLMfNBRERkFHMHH+5uF2Y+iIiIjGPy4ONs5kPTtGBvDhERkSmYOviwuScZy3dqyCtwBntziIiITMHkwYcr8yGyONEYERGRIUwdfERZLYiPcb0FnOuDiIjIGKYOPoSdI16IiIgMZfrgw6aPeOEU60RERIYwffBh1zMfrPkgIiIyBIMP93BbZj6IiIiMYfrgQx9uy5VtiYiIjGH64MPuWd+F3S5ERERGMH3woRecMvNBRERkDNMHH3Z35iODmQ8iIiJDmD748GQ+OMkYERGRIUwffNj1mg9OMkZERGQI0wcfHO1CRERkLNMHH1X0eT5Y80FERGQI0wcfNnfwwcwHERGRMUwffNjd3S6s+SAiIjKG6YMPm2dtF2Y+iIiIjGD64MOur2rL4IOIiMgQpg8+9MwHu12IiIiMYfrgQ898sOCUiIjIGKYPPvTMh6NAQ16+M9ibQ0REFPFMH3zoo10Esx9ERESBZ/rgIzrKirho19vAug8iIqLAM33wIez6RGMc8UJERBRwDD681ndh5oOIiCjwGHx4r2zLzAcREVHAMfhQ67twojEiIiKjMPjwynxksduFiIgo4Bh8FKr5YOaDiIgo0Bh8AKjiGe3CzAcREVGgMfjwrvlg5oOIiCjgTBd8RBdkn3ObnTUfREREhnHtdc1g/ypEf30H+uXK+i3XFr+yLUe7EBERBZx5Mh/JDWA5vRtVs/cCWSeKXdmWwQcREVHgmSf4SKwNrWZrddGyZ1HxmQ92uxAREQWceYIPAM5G56lzy+75xWY+uKotERFRmAYfBw4cwE033YQaNWogISEBHTp0wMqVKxFsWmNX8GEtEnycrflg5oOIiCjsCk5PnTqFfv364cILL8TPP/+MmjVrYvv27ahWrRqCTWvUDxossJzcCZzZr+pABDMfREREYRx8vPjii0hNTcXHH3/sua1JkyYICfHJOG1rgmpZO4Fd84HOI9TNdmY+iIiIwjf4+OGHHzB06FBcd911mDdvHurXr4/77rsPd955Z7GPz83NVSddWlqaOnc4HOrkT/J8xxLbquDD+edcFLS7Tt0ea9U8mQ9/v2Yo0NsUiW0rDdttrnabue1sN9sdCsqzPRZN01x7Xj+Jj49X5w8++KAKQFasWIExY8bg3Xffxa233nrO48eNG4fx48efc/vkyZNhs9ngbzXTNqDvnxOQHVMNv7WbCFgsOJULjPsjGtEWDa/0ZvaDiIiovLKysjBixAicOXMGSUlJxgYfsbGx6N69OxYvXuy57f7771dByJIlS3zKfEi3zfHjx8vc+IpEZbN/nYFLN4yCpSAPjnuWAjWa40y2A92fm6Mes2ncIMRERdYgIGn3zJkzMXjwYMTExMAs2G5ztdvMbWe72e5QIPvvlJQUn4IPv3e71K1bF23bti10W5s2bfDNN98U+/i4uDh1Kkre0EC8qQXWOGj1u8OydzFi9i0C6rRBksVVcCocTits8aHzx/SnQL2noY7tNh+ztp3tNpeYEGt3ebbF74f4MtJl69athW7btm0bGjVqhFChNR7guiBFp5KtibYi1p3t4OJyREREgeX34OOBBx7A0qVL8dxzz2HHjh2qduP999/HqFGjECr0+T6wawHgdBZa2ZbDbYmIiMIs+OjRowemT5+OKVOmoH379nj66acxceJEjBw5EqFCq9cViLED2SeBIxvUbXYOtyUiIgrfVW0vu+wydQpZUTFAo77AjpnArnlA3Y5nF5dj5oOIiCigImtYR3k0Pb9Q3Yc+xXoWMx9EREQBZd7go4m76HTPYqDAwcwHERGRQcwbfNTuACRUA/IygAN/cHE5IiIig5g3+LBaAc+ol3mwx3K0CxERkRHMG3wUqfuwxTHzQUREZARzBx9NLnCd71uG5CjXgjjMfBAREQWWuYOPGs2AxHpAQR6a521UN7HglIiIKLDMHXxYLJ6ul2bpq9Q5h9oSEREFlrmDD68htw1Or1DnzHwQEREFFoMPd/BR48wmJCETWXnMfBAREQUSg4/kBkD1ZrDAiV7WzcjIZeaDiIgokBh8CHfdR1/rRtZ8EBERBRiDD6+uFwk+WPNBREQUWAw+RGNX8NHKuh/xuSeCvTVEREQRjcGHsNdAXko7dbGjY22wt4aIiCiiMfhwc7qzHz20DcgvcAZ7c4iIiCIWgw+3qGauotN+1g3IcrDolIiIKFAYfLjFNO2PfM2KhtZjyD26M9ibQ0REFLEYfOjiErHB0lxddO6cH+ytISIiilgMPrysjuqozmP3LQj2phAREUUsBh9eNsR1Uef2g4sBTQv25hAREUUkBh9e9iS0RY4Wg9jsY8CxrcHeHCIioojE4MNLbLwNK5ytXFd2zQv25hAREUUkBh9ebLHRWOJ0TTaGXSw6JSIiCgQGH17scVFYpAcfuxcATs73QURE5G8MPopkPjZoTZAbZQdyzgCHONU6ERGRvzH48GKPjUIBorAnsavrBtZ9EBER+R2DDy+2uGh1vt3uDj52MvggIiLyNwYfRTIfYmNcZ9cNe5cC+bnB3SgiIqIIw+CjmMzHTjQE7DWB/Gxg/4pgbxYREVFEYfDhpUqcK/ORKavaNhngupFDbomIiPyKwUeR0S4iK88r+GDdBxERkV8x+PBidwcfmbn5QJPzXTceWAnkZgR1u4iIiCIJgw8vNne3i8p8VGsMJDcEnPnA3iXB3jQiIqKIweDDi90782GxAE31ug92vRAREfkLgw8vNvdQ28y8fNcNTS5wnbPug4iIyG8YfHixu4fa5jicKHBqQJPzXHccXg9knQzuxhEREUUIBh/FZD5ElmQ/EusANVsD0FwLzREREVGlMfjwEhdtRbTVcrboVHC+DyIiIr9i8OHFYrGcrfuQolOhD7ll3QcREZFfMPgooe7Dk/lo3A+wWIET24G0g8HdOCIiogjA4KOIczIfCdWAup1cl9n1QkREVGkMPkrIfHiG2wrWfRAREfkNg48SMx/ubpeidR+aFqQtIyIiigwMPoqwexaX88p8NOwNWGOAtP3AyZ1B2zYiIqJIwOCjCJve7eKd+Yi1A6k9XZc51ToREVGlMPgooopncTmvzId33QeH3BIREVUKg48ibPricvpQ26J1HzLTqdMZhC0jIiKKDAw+irC7C06z9KG2uvrdgBg7kHUCOLoxOBtHREQUARh8lFTzUTTzER0LNOrjuswht0RERBXG4KOEzIdnkjFvnGqdiIio0hh8+Frz4V10umcRUOAweMuIiIgiQ8CDjxdeeEEt2DZ27FiEA7s+2qW4zEedjkB8VSAvAzi42viNIyIiigABDT5WrFiB9957Dx07dkREZD6sVqDJea7LnO+DiIgotIKPjIwMjBw5Eh988AGqVauG8FvVtpjMh2DdBxERUaW49rQBMGrUKFx66aUYNGgQnnnmmRIfl5ubq066tLQ0de5wONTJn/TnK+1546yap+C02Mel9kMMAG3fcuRnpQExCQh1vrQ7ErHd5mq3mdvOdrPdoaA822PRNP+vlDZ16lQ8++yzqtslPj4eF1xwATp37oyJEyee89hx48Zh/Pjx59w+efJk2Gw2GO1EDvDU6mjEWjW81KuYrhdNw5CNY5HgOIVFzR/B8cR2hm8jERFRqMnKysKIESNw5swZJCUlGZv52LdvH8aMGYOZM2eqwKMsjz32GB588MFCmY/U1FQMGTKkzI2vSFQm2zV48GDExEj+4lwnMvPw1Oq5yHNaMGzYxbBaLec8Jip/BrDhK/SulQfnhZcg1PnS7kjEdpur3WZuO9vNdocCvefCF34PPlatWoWjR4+ia9euntsKCgowf/58vPnmm6qLJSrKNaJExMXFqVNR8oYG6k0t7bmr2s+WwThgRZWYYt6iZheq4CNqzwJEhdAfviyBfE9DGdttPmZtO9ttLjEh1u7ybIvfg4+BAwdi/fr1hW67/fbb0bp1azzyyCOFAo9QFBdthSQ7nJpruG0VdwFqsfN9HPwDyDkDxCcbvp1EREThyu/BR2JiItq3b1/oNrvdjho1apxzeyiSOUnssdFIz80vfritqJoKVG8KnNwJ7FkMtLrY4K0kIiIKX5zhtBi2uFKmWC+a/eA6L0RERKEx1Nbb3LlzEU5cc33kIqukzIc+38eqTzjfBxERUTkx81EMu2eWUx8yH0c3AhnHjNkwIiKiCMDgoxg298q2WbmlZD7sKUBtdw3Lbna9EBER+YrBRylTrJea+RCcap2IiKjcGHyUkvkoteBUsOiUiIio3Bh8FMPurvkoteBUNOoLWKKAU7uA03sN2TYiIqJwx+CjokNtRXwSUN89kyuzH0RERD5h8FEMu6+ZD8G6DyIionJh8FFawWlZmY+idR/+XyCYiIgo4jD4KIbd3e3iU+YjtRcQFQdkHAaObwv8xhEREYU5Bh/FsPkyyZguJh5o2Mt1mXUfREREZWLwUQy7L5OMFVv3EV7TyBMREQUDg49i2Nw1Hxm+1Hx4Bx+7FwJOHwMWIiIik2LwUVrmw5duF1GvCxCbCOScBg6vC+zGERERhTkGH6XWfPiYxYiKBhr3c11m3QcREVGpGHyUNtrF124Xwfk+iIiIfMLgo5R5PrIcBXA6fZy7o6k7+Ni7BMjPC+DWERERhTcGH8Wwu7tdZM6wnHwfu15qtgFsKYAjCziwMqDbR0REFM4YfBQjPsYKi8V1OdPX4bZWK1e5JSIi8gGDj2JYLBav9V3KU/fhDj5Y90FERFQiBh8lsLmH2/o814d33cf+FUBeZoC2jIiIKLwx+Cir6NTX4baiWhMgORVwOlyFp0RERHQOBh9lZD58WtlWJ4UiHHJLRERUKgYfJbB7aj7KOV06i06JiIhKxeCjjInGypX58A4+Dq0Fsk4GYMuIiIjCG4OPMhaXK3fmI6kukNJSZgkB9iwKzMYRERGFMQYfZSwul1meobY61n0QERGViMFHGYvLZfk6yZg31n0QERGViMFHGTUf5ZrnQ9e4vwx9AY5vBdIO+X/jiIiIwhiDj7IyHxXpdrFVB+p2cl3evcDPW0ZERBTeGHyUWfNRgW4XwanWiYiIisXgo6zRLhXpdvGean3XPNfyuERERKQw+ChBFXfwUeHMR8M+gDUGOLMPOLXLvxtHREQUxhh8lDG9eoVqPkSsHWjQw3WZo16IiIg8GHyUtbBcRYba6lj3QUREdA4GH2VkPio01Pacuo/5rPsgIiJyY/BRAntFF5bzVr87EGMDso4DRzf5bduIiIjCGYOPEtj0heXy8qFVNGsRHesqPBWs+yAiIlIYfJTA7s58SNyR43BW/IlY90FERFQIg48SJMS4Mh8VXlyuaN2HrHBbUInnISIiihAMPkpgtVo8s5xWasRLnY5AfDKQmwYcWuO/DSQiIgpTDD58mOW0UpkPaxTQ+DzX5Z1z/bRlRERE4YvBRyk8mY/KBB+iideQWyIiIpNj8OHDyrYZlel28a772LcMcOT4YcuIiIjCF4OPUtjdw20rvLicLqUlUKUOkJ8D7F/un40jIiIKUww+fMh8VHhxOZ3FwiG3REREbgw+fMl8VLbmo+hU60RERCbG4MOXzEdlaz6Envk4sArISav88xEREYUpBh+lqKKvbOuPzEfVhkC1JoBWAOxdUvnnIyIiClMMPnxY2dYvmQ/Bug8iIiIGH6Wx+zPzIVj3QURExODDl8xHRmWH2uoauzMfR9YDmcf985xERERhhsFHKezugtOsyg611VWpCdRq57q8e4F/npOIiMjswcfzzz+PHj16IDExEbVq1cJVV12FrVu3IhzZ3ENtM/2V+RCs+yAiIpPze/Axb948jBo1CkuXLsXMmTPhcDgwZMgQZGZmItzY/Z35EKz7ICIik3PtXf3ol19+KXT9k08+URmQVatWYcAA91F/uI128VfBqWjUF7BYgZN/Amf2A8kN/PfcREREZgw+ijpz5ow6r169erH35+bmqpMuLc01AZdkTOTkT/rz+fq87l4X1e3it22JsiGqbhdYD65C/o450DreiEArb7sjBdttrnabue1sN9sdCsqzPRZN07RAbYjT6cQVV1yB06dPY+HChcU+Zty4cRg/fvw5t0+ePBk2mw3BdDQbeHZNNBKiNLzQ039dL20OfoWWR/6HfdX64Y/Gd/vteYmIiIIlKysLI0aMUEmHpKSk4AUf9957L37++WcVeDRo0MDnzEdqaiqOHz9e5sZXJCqTOpTBgwcjJiamzMcfTc9FvwnzEGW1YPO4QbDIAnF+YNk1H9GTr4FWpQ7y71/vWngugMrb7kjBdpur3WZuO9vNdocC2X+npKT4FHwErNtl9OjRmDFjBubPn19i4CHi4uLUqSh5QwP1pvr63Ml2V1BQ4NTgtEQhPsbdD1NZTfoCUXGwZBxGTNoeIKUFjBDI9zSUsd3mY9a2s93mEhNi7S7Ptvh9tIskUiTwmD59OmbPno0mTZogXCV4BRt+HW4bkwCk9nRd3jnXf89LREQUBvwefMgw288//1zVbMhcH4cPH1an7OxshBvpbtEDEL8OtxUccktERCbl9+DjnXfeUf09F1xwAerWres5TZs2DeHIHheA4baiyflnZzp1Ov373ERERCHM7zUfAaxfDQqbmmgsz38r2+rqdQViE4HsU661Xup28u/zExERhSiu7WL0yra6qGjXhGOCU60TEZGJMPgog12f5dTfmQ/Bug8iIjIhBh9lsAUq8+G9yNyexUB+nv+fn4iIKAQx+PA58xGA4KNWO8BWA3BkAgf/8P/zExERhSAGHz4VnMpolwB0u1itQOPzXJdZ90FERCbB4MPHobZZgch8CNZ9EBGRyTD4CGbmw3u+j/3LgbyswLwGERFRCGHw4WPNR0AKTkX1pkBSA6AgD9i3NDCvQUREFEIYfPg4z0dAhtoKWdFW73ph3QcREZkAgw9faz4ClfnwHnLLug8iIjIBBh++1nwEKvPhHXwcWgNknw7c6xAREYUABh/BWljOW1I9oEYLQHMCexYF7nWIiIhCAIMPnzMfAQw+BOs+iIjIJBh8lMHuDj6yAjXU9py6DwYfREQU2Rh8lMGmd7sEOvOhZjq1AMe2AOlHAvtaREREQcTgowx2r8yHpmmBeyFbdaBOB9dljnohIqIIxuDDx4LTfKeGvAJnYF/MM9U6u16IiChyMfjwseBUZAVyuK33VOsMPoiIKIIx+ChDlNWC+Bhr4IfbioZ9AGs0cHovcGp3YF/LBPILnMhwBHsriIioKAYfPrAbMdGYiKsC1O/uuswht5XidGq4d/IaPL4qCnO3HQv25hARkRcGH+UZ8RLozEehug8WnVbGR4t2Ye6243BqFjw2fSNOZOQGe5OIiMiNwYcP7PqIl0BnPoqu8xLI0TURbNPBNEz4Zau6nBCl4XhGHh77dn1gRysRERnkx/WH8fkOK9Kyw7dfmcGHD2yxBmY+GvQAohOAzKOuOT+oXHIcBRg7bbUamXRRq5oY3a4AMVEW/LbpCL5auT/Ym0dEVCnH0nPx2PQNWHHMild/34FwxeDDB/Y4fa4PA4KP6DigUR/XZdZ9lNsLP2/BtiMZSKkSh+eubocGdmDMRc3VfeP/txF7T2QFexOJiCrszdnbke1wTfswZcU+bD6UhnDE4MMHdqMKTovreiGfzd16FJ8sdo0Seum6jqhhj1WX/9a/MXo2ro7MvAI8+OUaFDjZ/UJE4WffySxMXr5XXa5r0yA/ZeN+2BiWXcoMPspRcGpI5sN7vo/dC4ECg14zzElB6T++Wqcu39a3MS5sVavQcOlXru+EKnHRWLnnFN6d92cQt5SIqGJem7kNjgIN/ZrVwF2tC9Q0EMt2ncRP6w8j3JydQYtKZDc681G3ExCfDOScAQ6vBep3M+Z1w5RE/Y98sx7HM3LRsnYVPHpx63Mek1rdhicvb4uHv16nvsDnt6yJ9vWTYdZhyKey8nA0PVedpA/5aHoOjqa5LsupaU07nr6qPWKieHxCFAq2HE7D9DUH1OWHBjfHvrVHcFf/Jnhjzp947qfNuKh1LSS46xPDAYOPUFpcTmeNci00t2WGq+6DwUepJA35++YjiI2yYuINXRAfU/wX8C/dGmDW5qP4ZeNhjJ22BjP+3r/Ex4Z7cLHltAWnl+/Diax8d0CR4wo20nJVkCbLBZRm+e6TSEqIwb8uaWPYdhNRyV7+dasaAHlph7roUD8Z+9a6upS/WX0QB05nq4zuA4NbIlww+PCBXc985BmU+dDrPiT4kLqP8x407nXDzJ/HMvD0jE3q8j+HtULbekklPtZiseC5azpg1d5T2HE0Ay/+sgVPXt4Okeax7zbi281RwObNpT6uuj0WtRLjUDMxDrUS493ncch2FOClX7fi/fk70SW1Ki7uUNewbSeic63acxK/bz6qupAfHHI2wJBMhxwgjJr8hwo+ruveAA2q2RAOGHyUY6itYTUf3nUfe5cC+bmuUTBUSF6+E2OnrkGOw4n+zVNwR78mZf4f2eFO+EtH3P7xCny8aLdKVZ7XoiYixY/rDuHb1QdhgabqXupUTSgUYMjlWklxajRQaV0qZ7IdKviQbqqWdRLRrGYVQ9tBRGe7lV/82TVv0XXdGqjvosNxdn6PSzrUQe+m1bF050k8/9MWvDWyK8IBO3TLMdTWsJoPUbMVUKU2kJ8N7Ftu3OuGkdd+34b1B86gqi0GL1/XCVarxaf/Jzvlm3o3VJf/8dVanM7KQyQ4kpaDf3+3Xl0eVF/Dezd1wXNXd8DYQS0xslcjDG5bG51Sq6JuckKZtRz/HNoKvZpUR0ZuPu75bJVxXY5EVIgsDyHdoLHRVowZ1KLYjK5kcOXn78f1h7D4z+MIBww+QjXzYbFwyG0plu484Rm18sI1HVAnOb5c/19SlU1T7DiSlovHv9+ISKjzcAVSDrSrl4hhDVzzAFRUdJQVk0Z0UZmS7UczOEMsUZC+1y+5Z2u+tU8jdeBQnDZ1k9QBhnjqf5vUopqhjsGHD2SIpuE1H8IdfGRt/g0Tv1+CHYfPGPv6IepMlgMPTlujiq+u794Aw9qXvybBFhuNV2/orPpQ/7f2IL53V5GHq/8u2Y0F248jLtqKl6/tgGg/fLOlm0ZSuNFWC35YexCfuudQISJjzFh/CJsOpSExLhr3XeCaLLEkDw5uqbLAWw6nY4p7LpBQxpoPH3dUIsvg1POJWn1QQ17/2BqMPTYMztUWFMQlI8peA7DJqTqQUN11XuhyjbOX5Ry+dUeEAzn6/s/3G3DwTA4a17BVqmC0c2pV/P2i5pj4+3b857sN6NG4OupVLf7IIpRtP5KO53/e4snoNK9VBdv89Nzynjx2SRtV1PvMj5vRoUEyujWSz1RkHmVm5OUjKT4m2JsSVhbvOK66BLo3jszPRbA4Cpx45TdX1uOuAU1RzT1pYknk/ocGt1SZ3Jd/24bLOtYr8/8EE4MPH9g9k4wZk/nIzS/AJ4t2Y9LsnRjnHIAh1pVIsmTBCg3IPe06nfR9oqzo2CoYhHhEHXoFsKcUE6hU87rNHdTE2FxdPyHmuzUHVKZCMhav3dDZU49TUaMvbI45W49h7b7TeOjLtfjib718rh0JmaLbaWuQm+/EeS1ScHPvRijw88R0d/RrjNV7T2HGukO474s/MOPv56kC1kiy9XC6GjGw92QW3rixC4a1rxPsTQoLHy7YqYJSMerCZnhwcCv13aTKm7ZiH/acyEJKlVjc0b/sYnoxvGdDfLFsr8p+vDpzm5qrJ1Qx+ChH5kOK7wJ9VC/DqZ79cRN2u9cg+bT+I2h8eVu0qBGP0R/NwqHDB9EoIQfjB9dBg9gcIPskkHUCyDrlvuy+LpezTwGaE5a8DNiRARwuRyFSVJxXdqXauRmVQtkVub+Ga2K0AAYsMrXw49+56jPGDmyBLg2rVfo5pbZh4g2dccnrC7Bk5wl8tGgX/nZeU4SL12dtw8aDaYWKbgv8HCNLQduL13ZUP2gyRPn+Kavx2V97qvcu3Ml37qtV+/HE9xvUqCnx9yl/4P2bu+PC1mdnyaVzSc2VrKWke2vOn9hwIE0Fb8k2Zo8qIzuvAG/M2u45QPL1IEu+k5INHv7BUnyxbA9G9Gqo6kFCEYOPcmU+Ahd8bDuSrlLb0m8v5Mjy4aGt8JeuDTxH4m/eOQw3/d8yzDpwBqt/j8XkO3uhdZ1SPlhOJ5BzGo70o1gy60f07dwK0XlprgDFE7S4gxTP5ZNAQR5QkAukH3SdfGWJKhyIyPBgOUkgo1/2XI8FouN9vi/fEotXpm9G9bxs9G2Qgvt6VQNyM1z3W6MrFfQ0SbHjP5e1wb+nb8CEX7eqobet6iQi1K3cfRLvzHVlwGRUS+2k8hXdlof8+L17U1dc+eYiFaRJWre4mWTDiXyfpbvt2z9c9T6SOUqMj1ZTVd/9+Sr8363dI2oYtr8XN5PPgHhgUEs0TrHhkW/WYd62Y7j8zYV4/5Zupf82UalkjSqZFLBBtQQM7+UameerPs1qqInIZOSLrPsy9a7e6gAi1DD4KEfmQ+bUlzS39G/6iwzzlOm+P1+2Vy14JrN0/vW8Jhh1YXNPoatOjiY+/2sv3PzRMqzbfwYjPlhWegBitbq7UBJxyt4cWouhQEwZRyRSxZmXUSRAKSar4rksgctJwJEJaAVA1nHXyc/knZgoFyTbL0//stedFmuxQUt0VCwGZGQj6vjbQEy8+z73Y+Sx6nGuyyOi4hBX5xi2HsvG3E9noen5LRATHeMKbKz6eRQQpV92X/fcFw1E6bfr/yfK6z7v617PV8EfhfQcBx74co1aWOqarvVxiQETgTWvlYgJf+nkmdCoS8OqGNouPLsnJNiXLiTJ5Ehs/9CQVrj3/GYo0DTkF/yB3zYdwZ3/XYlPbu+J3k2l8or0TJHUSL3uPiqXAyT5rRJSa3T3Z6tU19XVby1WiztK3QGVv6D+nbk7PEWkcdHln4X5sUtaq1mf9XVfLu0YehMFMvgox1Bb/WgpVnZclSRDoaRvTvrlZEInMbRdbVUw2KiGvcT/JwHIZ3cUDkCkTsFvqTXZGcYluk7VXEO3fOLQu4DcQYusSyOTo0kWJT8HyHdnU/L1U04p98n/O3s5Nzcbp9LSEQsHkqILEK05AKdXFkpzuuZDkRPOjgiS3brqmMnaWXazZfp1uSCxmTzNLzCGJ1gpK5gpfDpyIhfPpTsQbYtB99wUYEqMKwizRiEKFnQ9dARRP8xwBT3qdj3YiXKfW4tc9+32S63RyG99GLO3ncDvXy5D54vbonaSrZLPbSn8mEKXvf5PocuWCgduX63ch8fd3SwylHjS8C7o5Q4wrLCoIcYyt4nUAt3xyQrVxRSpRbblDTxe/m2r6l4Rj13cGnef38xzf7t6yfjf6P64f+pqlcEdPXk11u8/owKUSOiiM8p78/9EWk6+Wqfqys71K/QcMsvpPec3U0GidOOH4rovDD58IBMySbZDsh4y3LZqJWevXbD9mBqLLfMniNZ1EvHEZW3Rt3mKT/+/aAAy8kM/ByAVIZmFmHpAkn+PdGRyq0vfWIDduVm4vFM9vHFjZ9dOx1lwTpBSKJjJz0F+bhZWLluE7p07IBoF7vu8gxv3473+34FT6Vi585h6fJ/GyaieYHUFOnKSQk79slMCoAL37e5gSL/u9Lqu3ydZoeLoz1dOcqzZXH5LpEyhSO2x/MynyoVTSxAIV8pJj7+NCtKKowcqXkFJtMWCi/MLEL014ZzAxWmxqrVuuuYU4AeLFfGJsahXzY7omYUDpDhLFD60WLElOQMnsgqQ/VEU0lKrIckW735NS+HgSN8GT7BkDcB9esBVzP3WKFicTtQ7tRaWzQ5XVs/7/3pOlhIue50kDC/mPpnh5YOFu/HzygNobLHgvgtb4vqOUcBpGdJ59vHVLFZ8cn0TvDlHw8eLd2PK/HXYuf8QJlzXGdXscV6vUXS7Qq9bIBiOpuWoujPx8NDWlSreleDj61X7Q3bdFwYfPpIukJP5eZUabrvreKaKQqWoVFSzxeDBIa0wvEdquY8MVAAiXTD/F0IBSACOtCRIk+Lb+lUT8MxV7c/2XcqPcaxEgSVHgprDgSNbsqC1uaTs7iY3Oc54/eu1+HLlftQ/loBfxp6HRH8MvZTuLBWcOMoOZjwBzbnBzOmMTDz9w3rk5OZhaJsUXNGhputxEtyocycK8vOwaeMGtG3dElHydqn/6/R6zNnHFr5exu1el/McDqzddxLOgnzUtMegSY14WDz3F/da3rfne72Gs/Dr6ZfV7q6s99T9f4U7tpPmqrgoK/Och8s3TCo4aupfNUk4ur6K55C4Tg3i1g8W9yOkyQ95D7kQoKlY5H29S076IKfF7lMx5C0bIye9BEnKxl734RWKBj2eIMXrvFDgYkE0LBiam4foHY8UzogV+/+LBj7uALbIcxZ/mw/P2f8BIFX9FSrsjdnbVUaua8OqGNSmcgXPob7uC4OPcnS9nMws/0Rj0r2yYvcp/LT+EKau2KvqRmTSppv7NMLYgS0rVRWenFA4ABnxwVJMvrO3XwMQyTys2H1S9eca9cHNcRSoNUo+XbJbtUu+569c30m11whPXN5OFVXuO5mt6gKkelzaXynSCOlGkVMlgrGxn6zA3OwY9TceNrwfiptNzOlwYOexn9C69yWI8jHoKi/ZwTt3nsCID5eh4JSGpwe0w819GvvvBSRYOycI0oMXd9Bxzu0FcDhyMX/eXAzo3x8xEnlpTszadAjvz90OR34+qtui8cDAZmhXx17K8+lBkBM5eXl4f94O7DuZgcRYK+4e0Bi1q8QU2T79/7ufQ52Xdr/7vgrd7yz2PqezACeOH0ON6lVdM0d63y+BnOf/eV8u4zo0aJoTmTl5yHXkq6H+9hgLYt3v67n/9+z/q8Af/OzzlYPEDyrGST+NoOsyslL/fc+JTExdvk9dfmRYa78Uicq6L7JMgtR+PPfTZrw9MnRWSGfw4SN7OSYak7qQ+duOqaK12VuOqimvdee3rInHL5OJoPwzmiJQAcimg2lqqNb3aw56hhjL4kXXdm2gVjktWgzrDwdPZ+PzpXswdcU+nMx0rbci3V2yzoiRRX/Stteu74wb31+q+q6HvDZP9b3eP7CFGhkTLFKUPHfrMfWevH5jZ78WPleE1Ek8Oqw1nv1pM56asQnt6iejqx+GPyvedSDl4XAgI347UKsNsjWrGkL71So5YGiqFh98/obO5ZqjRHZst7V34KYPXd+vHxbFYdrdvf260J5812TI9MLtx9XINik617t6JYCKjY5SO/yzt7lOMpttTLTrdvkoOPP24NHrB6NalQS/TbomawVNWb5P/TlkuPX13VWHXtm8ApMzmbn4x1ersXD7URXAyFw0Dw1ujhiLV8DhHbjomS/vAEc957kBkiPfgUUL5qNfv36u9YrUEgDFBFvFBmDejy0cdJ37/4t//UL/v3bl5tR4deY25Ds1tY/Qa5AqSwKYcVe0U13XUngq6770beZb936gMfjwkc093LakuT6Opedi1uYjmLnpCBbsOK7qQ3TSvXJR69q4qku9gAzd0wOQW/5vGdZWIgCRseX/W3cQk5ftxZp9Z48k5Mf6eEauWjVRTlKsN6xdHVzTtQH6NU+pVL+kHM3Lc8rU3b9tOqxGb4h6yfG4qU8j3NA9FTWqGD+hlczW+P3ofnht5nZVNT599QE1xfjVXerj/otaoGENY9OXfx7LUF12+lFRy9qhMRT4b+c1wR97T+HnDYcxSk1A1j8of6+ipJ5q7JfrsO2IazSLDAe978LmFfqsyoyn/72jJ4Z/sAybD6Wp79eXd/cptTDcF/Jcr/++Hb9sPAz/iMKMl+apz+hNvRtVaqirjLyTobNSMyBvmcwhI9/3cgePiEJyUgzevb0/Jv6+DZNm78C7S49i9ZF8NXW/rK5cKQ4Hztj2AnU7+dy1Goo2HjyjDvSEFOj6k77uy2dL96hubPmOhkIBMIMPH9n1zIdXt4vsECTYkJP8AHuvu9Wwuk2tIiqn7o2qBfyPLQHIf4sEIF/8rTfa1kvyadihBBzf/LEf6Tmu4Eq6hmQYpUxS06dpDRxKy8F3qw/gm1X7sfN4Jr5bc1CdaifF4aou9VVGpDw7ROnOkdlK/7t4D7YeSffcLq91a9/Gqr8z2F8Qqd7/8NbuWLf/tBpeKFks+TGWQETmXxl9UXOkVrcZMs3yA9PWqL7gfs1r4Pa+fuze8MOR1YS/dFR/w53HMjFm6hp8ekdPn3bycmQtcxkcOJ2F/aeyVWHcgVPZqp0yaVp1e6w6r2aLdZ3srstyW2nDD1ccs+DRd5ci2+FUgbNkiSp7tFfVFovP/9pTZcMksJFRZpIBqUhXpMymKpkOORLV99MyJPXO85qo+iL5e8vBS577XK6fvU0r5jYn0rPy8M3ynTiaU4DPl+5Vp26NqqnVmy9uXxfxMVHl6ip++Ot16nMuf8dXr+9U4VEXOnkeGc7cvn6ymklYugEun7QQ797UTa20bHYv/+qaRl2K6uU98jcZsisHTzJR4OTle3GLP7tIK4jBRzmH20pGQP6AMzcdxp/HChe1dWyQjMFtamNwu9poVTvR8IldigYgIz90BSAtaiYUW1fx84ZDKuiQmhSdmtSmZ0NVnCQLi+mk4FPG8993QTP1HsjETPJhllVh35u3U53a109SQcgVneqVePQrRbefLdmDr1bt8wQ6CTFRaq4KCTpC5YjeW8cGVfHRbT1Uu2VOFplIadrKfSpYu657qgpC5P0JlEmztquUf1J8tGcW01AiO0zZicgEZAt3HMerM7eqSn3ZMR464woo9rsDiwNe53Kf1EBVhD02SgUE3gGKXD5yJhs/73ANA5JAbeINXfw2Fbx8pr+4sxdufG+pCsClyHvaXX18XlFZ1uCZOGu7qv+SAxX5eZD5WcYMbFHpz73D4UCrvO2o3qYXpq08iF83HsaqPafUSY525XM6omdDNC6j21ACjwe+XKuWMJADkDeGd/HrHDJyQNNslB13fbZKBavXvbdEFZJ7T6ZoNst3nVTDuiVAkyAhENS6L0Na4onvN+KV37bh8hBY98Wihdg62WlpaUhOTsaZM2eQlOTfkRvyBf3pp59wySWXIKacKTo58pQjAW/y5ZTZ5Ia0rY1BbWuXuNyx0WTekFs+Wq7WK5Eun09v645dqxeodu89nYspy/bi6z/2e2pR5EM/sHUtjOzdCOc1T/H5R0DWoJmz5Ri+/WO/ygpIf6X+vlzQqqZK0w5sUwsxVqvaYUsBqdQs6GRhOInAr+3WICDFpJX5e5dGftAlhazPRiv98jf2aIj7Lmzm98+AvNZ17y5W3VEyH4UcGQWr3WWRlYEl8yEkIyZZjbJ+XeSzVycpXgVv9aslqHPp4pSJlk5l5eFkpkNNxCeXT2W5LutdcyWxQFMLBo4ZFJh1Rg6fycH17y1Rk2k1rWlXM0h6B+pF7TgqmY4dmLHuoOf9kELAMQNb+m0m3aJ/cxmyKWuDyOqmsgijTmZxlRR8cZlFyaTI1PnShSaf6TdHdA3YJHJqkrxpa1WXpv6bIXOu1EqKV58dma1XTnKbfllul98J74O6YH3W/UXTNFz37hKs3HNKZZllpmJfVKTdElheNmmhOniWuptArPtSnv03gw8ffTB/pyqsk2JE2bEOaVdHnYfqCphFA5CBtXOwHylYuutslkPqKm7s2VAVkfl69FYSKRCVoyUJRCTropMfi6SEaDVyRMjvxgUta6osx4AWNQN6tBPoHyY5YpFMiIyMEVIMKEeXkh2SH9HKkq6pS95YoBaXuqpzPUy8sYtP/y+YP8gynbNMDa2Lj7GqlYIlqJCs2tkgw6bOayfGlat7TbpqJGOmApOsPFdgkukKVOSUnu1A1fSd+PuNgW37/lNZuOG9pSqDI5NBTb2rj8q8eJNuWVmfQzKE+q+sTCQ4dlBLvw+JL+lvLrUbc7YcxefL9qgDAH07JOC7sWeqCprluy8HEjIpmHQhS8HrOzd1xcA2tRFI8rd8a84OTJqzo1CNXGnkO6aCk0RXQJJSJQa5R3bhkRH+K7Q10qzNR/DXT1eq4uH5/7zQ5yUSKvodX/LnCbXui/zsygKRvnTLlweDjwD8KEvUKEc68oNZkelugx2A6ORDd2GrWirKvqBVrYAcGUp6+dvVBzD9jwM4nOY66pI1M6R4VArhykr9+otRO2H5Qr/2+zYVjAj5IZF2yiQ/lUn5P/rNOjXyR4LEn8cO8Dk7FMzgQ3YoS3edUEG6BBqyQzay+9HItsvQSMmASNdj27pJmHJnbzV0XroWpatMapr0LI3Ufo0d1ELVEQWr3bIwo/T3f7liH064R5PJ91+yIFLLJpk82bm/f3M39dtgFMm4SEG7vI9H0nJU1ka/fCQ91309R2W+Slt/yx+FtkZ/Vy55Y4HKRMhvRXnWSqrM5/y+L1apeiMZguvvdV/Ks/9mzYeP5OisqR+H1xnBNQqmJ+76dAW2HjiJEX2bYUTvxgGtTxAtaieqERn/GNIKy3aeUD8aF7au6VkjJ9JI11vvpr2xWIKQmdtUCvX/Fu7Cx4t2uTM/MSr4kizZ2XP3Za/7kryuS42HBB6uOU46GzbHSWVJJitUhvIFmox2kVFlkgHZdCgNt3y0TA2hl6BDMg5CduyS6QhEEWF5SXG0fC8lCPplw2G1vIMEzL9uPOLJUn14Sw/0b2Hs30+GyEp3ZVldllKnJqMKj6bn4PAZV3By4FQm/rdqd6FCWynwlyDk4g51AnKgKFMCqJGNm4/iwKksVQskBxk1i54nxqnRPDWqxLqGARfxg7sAVL7vsq6QUWTisVmbj6rAMz03P2jZ+8jcG5CHfLA+u6OHK0oe2NzQI2H5cPs6ZXy4k6MHGXbct1kNdQQpmZDVe0+rwKu0I7ay/K1/ExXcUGiS+T5kZuEb31+iuhv1LscLW9VUQUcojuSQHbKMXpGTPtJt9b7Taq2WUF5ET0bsSADlPcJMMgAdCv5Etda9MHXlATW3kgT/cnpqRqwqnB/Zs1GlhsZL58DGg2mqPkW6peSyt6IDD4ojGUAJSlISXec1E+M8o50k61GZySbLS0Zo/Xj/eWhW0x7U1W4ZfBD5kXyZB7SsqQr75CjtdLYDadkOVaeQluNQC0Z5X0/3XHfdp86z85HtKFBTLP/Dz2P+yf+kaPTzv/VSxZoyxF4mo+vir8nWAkxG2cgkVOFM9p8S9J/fuo7KhsgsoVJoK12+Mgrv/fk7VX2ZZENkgTVfupqlBkbmH/p90xEVdBzyKtqV15PsyqA2tVVGS+rd5Lt+LCMXx93n6np6rurekiyYPEZOW11JJg8JQm7vZ/yw10rP2OwHDD6IAhSESNFpRQtPpR9cRgAE88iEfCe1HLMeuiDYm2F6UrA5ZlALjLqwGWZJoe3SPSoTKcW2cpL6KZlK4IaeqeeMUJLi5Tlbj+L3TUfVY70nlJTpAAa0TFEBhwQwvk6kJ3UdUgjtHZBIfcux9FyVEZWpCSK1O7osAWv1W2+9hZdeegmHDx9Gp06dMGnSJPTs2TNQL0cUUYrrIyYi32v0ZJiwnHYfz3QV2q7cp4YdvzJzm1pqXu67tlt9Nd+IdKdIV41eqyNkmK+M+BnctpaqYyrPRG3eNVASqMipdWBGLYetgAQf06ZNw4MPPoh3330XvXr1wsSJEzF06FBs3boVtWoZV0VNRETmJqPrpMhSJvCSBStl2LHUY/24/pA6eWtdJ1FlN2R0Uof6yaad+Cxsg49XX30Vd955J26//XZ1XYKQH3/8ER999BEeffTRQo/Nzc1VJ++hOnohkZz8SX8+fz9vqGO72W6zMGvb2e6y2y15iys61lYnGZ00ZcV+NbNosxQ7LmpdU51SvabLLyjIR0H5FjGH2f/ejnJsj9/n+cjLy4PNZsPXX3+Nq666ynP7rbfeitOnT+P7778v9Phx48Zh/Pjx5zzP5MmT1fMQERFR6MvKysKIESOCM8/H8ePHUVBQgNq1C8+OJ9e3bNlyzuMfe+wx1UXjnflITU3FkCFDAjLJ2MyZMzF48OCwnIq3othuttsszNp2tpvtDgV6z4Uvgl5mGxcXp05FyRsaqDc1kM8dythuczFru83cdrbbXGJCrN3l2Ra/l9SnpKQgKioKR44UHtAs1+vUYbkvERGR2fk9+IiNjUW3bt0wa9Ysz21Op1Nd79Onj79fjoiIiMJMQLpdpIZDCky7d++u5vaQobaZmZme0S9ERERkXgEJPm644QYcO3YMTzzxhJpkrHPnzvjll1/OKUIlIiIi8wlYweno0aPViYiIiMgb53AmIiIiQzH4ICIiIkMx+CAiIiJDMfggIiIiQzH4ICIiIkMx+CAiIiJDMfggIiIiQwV9YbmiNE0r9+p45VkJUJb8lecOpcV4Ao3tZrvNwqxtZ7vZ7lCg77f1/XhYBR/p6enqPDU1NdibQkRERBXYjycnJ5f6GIvmS4hiIFmE7uDBg0hMTITFYvF7VCZBzb59+5CUlASzYLvZbrMwa9vZbrY7FEg4IYFHvXr1YLVawyvzIRvcoEGDgL6G/LFC6Q9mFLbbXMzabjO3ne02l6QQbHdZGQ8dC06JiIjIUAw+iIiIyFCmCj7i4uLw5JNPqnMzYbvZbrMwa9vZbrY73IRcwSkRERFFNlNlPoiIiCj4GHwQERGRoRh8EBERkaEYfBAREZGhTBN8vPXWW2jcuDHi4+PRq1cvLF++HOHk+eefR48ePdTMr7Vq1cJVV12FrVu3FnpMTk4ORo0ahRo1aqBKlSq49tprceTIkUKP2bt3Ly699FLYbDb1PA8//DDy8/MLPWbu3Lno2rWrqqRu3rw5PvnkE4SCF154Qc16O3bsWFO0+cCBA7jppptU2xISEtChQwesXLnSc7/Uij/xxBOoW7euun/QoEHYvn17oec4efIkRo4cqSYiqlq1Kv76178iIyOj0GPWrVuH8847T303ZNbECRMmIFgKCgrw+OOPo0mTJqpNzZo1w9NPP11orYhIaPf8+fNx+eWXq5kg5TP93XffFbrfyDZ+9dVXaN26tXqMfMZ++uknBKvtsmbJI488orbDbrerx9xyyy1q1utwb3tZf3Nv99xzj3rMxIkTw77dJdJMYOrUqVpsbKz20UcfaRs3btTuvPNOrWrVqtqRI0e0cDF06FDt448/1jZs2KCtWbNGu+SSS7SGDRtqGRkZnsfcc889WmpqqjZr1ixt5cqVWu/evbW+fft67s/Pz9fat2+vDRo0SFu9erX2008/aSkpKdpjjz3meczOnTs1m82mPfjgg9qmTZu0SZMmaVFRUdovv/yiBdPy5cu1xo0bax07dtTGjBkT8W0+efKk1qhRI+22227Tli1bprbx119/1Xbs2OF5zAsvvKAlJydr3333nbZ27Vrtiiuu0Jo0aaJlZ2d7HjNs2DCtU6dO2tKlS7UFCxZozZs314YPH+65/8yZM1rt2rW1kSNHqs/WlClTtISEBO29997TguHZZ5/VatSooc2YMUPbtWuX9tVXX2lVqlTRXn/99Yhqt3wO//3vf2vffvutRFXa9OnTC91vVBsXLVqkPusTJkxQn/3//Oc/WkxMjLZ+/fqgtP306dPquzpt2jRty5Yt2pIlS7SePXtq3bp1K/Qc4dj2sv7mOrlf2lavXj3ttdde08K93SUxRfAhH95Ro0Z5rhcUFKg/7PPPP6+Fq6NHj6oP8Lx58zxfWvkAyY+1bvPmzeox8gXWP/xWq1U7fPiw5zHvvPOOlpSUpOXm5qrr//znP7V27doVeq0bbrhBBT/Bkp6errVo0UKbOXOmdv7553uCj0hu8yOPPKL179+/xPudTqdWp04d7aWXXvLcJu9HXFyc+sER8sMi78WKFSs8j/n55581i8WiHThwQF1/++23tWrVqnneC/21W7VqpQXDpZdeqt1xxx2FbrvmmmvUj2mktrvojsjINl5//fXqPffWq1cv7e6779aMUNpO2PvAQx63Z8+eiGk7Smj3/v37tfr166vAQQ4+vIOPSGi3t4jvdsnLy8OqVatU2tJ7/Ri5vmTJEoSrM2fOqPPq1aurc2mjpCy92ylptYYNG3raKeeSYqtdu7bnMUOHDlWLFG3cuNHzGO/n0B8TzPdKulWk26TodkVym3/44Qd0794d1113neoq6tKlCz744APP/bt27cLhw4cLbbesqSBdit5tl9SsPI9OHi+f/2XLlnkeM2DAAMTGxhZqu3TpnTp1Ckbr27cvZs2ahW3btqnra9euxcKFC3HxxRdHdLu9GdnGUPzsF/dbJ10Q0t5IbrvT6cTNN9+suoXbtWt3zv2R1u6IDz6OHz+u+pG9dz5CrssXPBzJh1TqHvr164f27dur26Qt8oHTv6DFtVPOi3sf9PtKe4zsrLOzs2G0qVOn4o8//lA1L0VFapvFzp078c4776BFixb49ddfce+99+L+++/Hp59+WmjbS/tcy7kELt6io6NVwFqe98dIjz76KG688UYVRMbExKigSz7r0s8dye32ZmQbS3pMsN8D75ouqQEZPny4ZwG1SG37iy++qNoh3/PiRFq7Q25VW/ItE7BhwwZ1RBjJZLnoMWPGYObMmaowykwkwJQjnOeee05dl52w/M3fffdd3HrrrYhUX375Jb744gtMnjxZHf2tWbNGBR9SpBfJ7aZzSVbz+uuvV8W3EohHslWrVuH1119XB1qS5TGDiM98pKSkICoq6pwREHK9Tp06CDejR4/GjBkzMGfOHDRo0MBzu7RFuphOnz5dYjvlvLj3Qb+vtMfIUYdU3Rv9hTx69KgahSIRvpzmzZuHN954Q12WaD3S2qyTUQ5t27YtdFubNm3UyB3vbS/tcy3n8v55k1E+UjFfnvfHSJJy1rMf0l0maegHHnjAk/mK1HZ7M7KNJT0m2O+BHnjs2bNHHXx4LxsfiW1fsGCBapN0Geu/ddL2hx56SI3SjMR2R3zwIWn5bt26qX5k76NKud6nTx+EC4n+JfCYPn06Zs+erYYiepM2Sprau53Szyc7K72dcr5+/fpCH2D9i63v6OQx3s+hPyYY79XAgQPV9srRr36SbICk4PXLkdZmnXSpFR1KLXUQjRo1Upfl7y8/Ft7bLd1E0vfr3XYJzCSI08lnRz7/Uj+gP0aGAMqPvXfbW7VqhWrVqsFoWVlZqg/bmxw8yDZHcru9GdnGUPzs64GHDC3+/fff1VBzb5HY9ptvvlkNkfX+rZNsnwTj0u0ake3WTDLUVirFP/nkE1UxfNddd6mhtt4jIELdvffeq4bezZ07Vzt06JDnlJWVVWjYqQy/nT17thp22qdPH3UqOux0yJAhariuDCWtWbNmscNOH374YTVy5K233gr6sFNv3qNdIrnNUuEfHR2thp5u375d++KLL9Q2fv7554WGY8rn+Pvvv9fWrVunXXnllcUOx+zSpYsarrtw4UI1ash7aJ6MopCheTfffLOqsJfvirxOsIba3nrrraraXx9qK8MOZWi0jEiKpHbLCC4Z+i0n+Rl+9dVX1WV9RIdRbZRhl/I5e/nll9Vn/8knnwz4sMvS2p6Xl6eGFTdo0EB9X71/67xHcIRj28v6mxdVdLRLuLa7JKYIPoTM3SA7KZnvQ4beyjjpcCIf1uJOMveHTn6Y7rvvPjXUSj5wV199tfrSetu9e7d28cUXq7Hf8qP+0EMPaQ6Ho9Bj5syZo3Xu3Fm9V02bNi30GqEWfERym//3v/+pwEkC59atW2vvv/9+oftlSObjjz+ufmzkMQMHDtS2bt1a6DEnTpxQP04yV4YML7799tvVj6A3mUdChvXKc8iOX3Z8wZKWlqb+vvJdjY+PV38LmRvBe8cTCe2Wz1tx32cJvoxu45dffqm1bNlSffZlyPmPP/4YtLZLwFnSb538v3Bue1l/c1+Cj3Bsd0ks8o+xuRYiIiIys4iv+SAiIqLQwuCDiIiIDMXgg4iIiAzF4IOIiIgMxeCDiIiIDMXgg4iIiAzF4IOIiIgMxeCDiIiIDMXgg4iIiAzF4IOI/O62227DVVddFezNIKIQxeCDiIiIDMXgg4gq7Ouvv0aHDh2QkJCglj4fNGiQWgb8008/xffffw+LxaJOc+fOVY/ft2+fWi69atWqqF69Oq688krs3r37nIzJ+PHjUbNmTSQlJeGee+5BXl5eEFtJRP4W7fdnJCJTOHToEIYPH44JEybg6quvRnp6OhYsWIBbbrkFe/fuRVpaGj7++GP1WAk0HA4Hhg4dij59+qjHRUdH45lnnsGwYcOwbt06xMbGqsfOmjUL8fHxKmCRwOT2229Xgc2zzz4b5BYTkb8w+CCiCgcf+fn5uOaaa9CoUSN1m2RBhGRCcnNzUadOHc/jP//8czidTnz44YcqGyIkOJEsiAQaQ4YMUbdJEPLRRx/BZrOhXbt2eOqpp1Q25emnn4bVymQtUSTgN5mIKqRTp04YOHCgCjiuu+46fPDBBzh16lSJj1+7di127NiBxMREVKlSRZ0kI5KTk4M///yz0PNK4KGTTElGRobqsiGiyMDMBxFVSFRUFGbOnInFixfjt99+w6RJk/Dvf/8by5YtK/bxEkB069YNX3zxxTn3SX0HEZkHgw8iqjDpPunXr586PfHEE6r7Zfr06arrpKCgoNBju3btimnTpqFWrVqqkLS0DEl2drbquhFLly5VWZLU1NSAt4eIjMFuFyKqEMlwPPfcc1i5cqUqMP32229x7NgxtGnTBo0bN1ZFpFu3bsXx48dVsenIkSORkpKiRrhIwemuXbtUrcf999+P/fv3e55XRrb89a9/xaZNm/DTTz/hySefxOjRo1nvQRRBmPkgogqR7MX8+fMxceJENbJFsh6vvPIKLr74YnTv3l0FFnIu3S1z5szBBRdcoB7/yCOPqCJVGR1Tv359VTfinQmR6y1atMCAAQNU0aqMqBk3blxQ20pE/mXRNE3z83MSEVWIzPNx+vRpfPfdd8HeFCIKIOYxiYiIyFAMPoiIiMhQ7HYhIiIiQzHzQURERIZi8EFERESGYvBBREREhmLwQURERIZi8EFERESGYvBBREREhmLwQURERIZi8EFEREQw0v8DpYG/aWdokTAAAAAASUVORK5CYII="
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "execution_count": 20
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 测试集"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T03:16:55.432078Z",
     "start_time": "2025-01-17T03:16:55.383962Z"
    }
   },
   "source": [
    "model.eval()\n",
    "loss = evaluating(model, val_loader, loss_fct)\n",
    "print(f\"loss:     {loss:.4f}\")"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss:     0.4662\n"
     ]
    }
   ],
   "execution_count": 21
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 拿到中间输出"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T03:16:55.437408Z",
     "start_time": "2025-01-17T03:16:55.432078Z"
    }
   },
   "source": [
    "logits, deep_output = model(train_ds[:][0].to(device), return_deep_output=True)"
   ],
   "outputs": [],
   "execution_count": 22
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T03:16:55.440315Z",
     "start_time": "2025-01-17T03:16:55.437408Z"
    }
   },
   "source": [
    "deep_output.shape"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([11610, 30])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 23
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-17T03:16:55.464914Z",
     "start_time": "2025-01-17T03:16:55.440315Z"
    }
   },
   "source": [
    "# 从这看到deep部分抽取到的特征分布，有些特征没那么重要，或许可以消减一些神经元\n",
    "plt.imshow(deep_output.cpu().detach().numpy().mean(axis=0).reshape(1, -1))\n",
    "plt.yticks([])\n",
    "plt.show()"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ],
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgMAAAA8CAYAAAD7aLkGAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAACVJJREFUeJzt3XtIlHsaB/BnRh3N0souXvLSxS6cLu6eyorI2mPYZYluf3T7w9owKosuVFKLWRAIBUsULe0fS/1TUnKyKM7CRpkRaC1FRFCSEahHTWrXS06mzvyW57c7g3OO56Q+b73O+X0/MEyT7/vO4+Mz7zzzvr93fg6llCIAAAAwltPuAAAAAMBeaAYAAAAMh2YAAADAcGgGAAAADIdmAAAAwHBoBgAAAAyHZgAAAMBwaAYAAAAMF9qbhbxeL9XV1VFUVBQ5HI4vHxUAAACI8fcKtra2UkJCAjmdTlkzwI1AUlKSPCoAAAD46mpqaigxMVHWDPARAfbtH/9MIWER/Qpk6L9+JKm2b+LF22icGSZaf+zfKsUx/PjXWNH6cRvkMdTmzRGtn/L31+IY/vPdONH6HVHyo1Rx/6gWrV//l2hxDPF5btH6qrVNHAMNk/0e26/9UxzC+UWzxdtQHq9sAx6POIbvnz4Srb/mm9+LY2hb9jvR+lEv/y2OwSGsy/d/SBHH4HHJ1v84Ur6P6Zoqy0PKn57Lnp866QH94H8fFzUDvlMD3AiE9rMZCHWG92u9gG3087m7C4mQNQOhDpc8hkhZLkIdYfIYwmW5DHVakAeXLIYQl/yFKq1L6d/yfzHI3oCUs1McA4XIfo/IqBBxCFa8tpRD2Aw45M1AdJTT9te3dF8ZKqwH5hDWpXT/oAlLKiRcvo/xRnrsrYf/zz70uVP8GEAIAABgODQDAAAAhkMzAAAAYDg0AwAAAIZDMwAAAGA4NAMAAACGQzMAAABgODQDAAAAhkMzAAAAYDg0AwAAAIZDMwAAAGA4NAMAAACGC+3tfMjM09ne7yfq8n7q97r+bQie38fTLps0okt1yGNwy3LRpeQT03g+yXLZ5bUgDx2yGDwdFswoJqxL6d/SihiUBX8L8shicLfKJ/ix4rWllHCiIiX/PVpavba/vqX7yi5hPTCHsC6l+we9Den6nyyYqMjdbms98KyF3d/Hf4lDfW4Jnu62tpaSkpJEAQEAAIA9ampqKDExUdYMeL1eqqur0/Mh9zQNYktLi24W+Mmio+Xzu5sMubQOcmkN5NE6yKV1kMve4bf41tZWSkhIIKfTKTtNwBv4tY7Ch/8g+KNYA7m0DnJpDeTROsildZDLzxs6dOhnl8EAQgAAAMOhGQAAADCcJc1AeHg4FRQU6HuQQS6tg1xaA3m0DnJpHeTSWr0aQAgAAAC/XThNAAAAYDg0AwAAAIZDMwAAAGA4NAMAAACGs6QZOHfuHI0dO5YiIiJozpw59OjRIys2a5Rjx47pb3fsfpsyZYrdYQ149+/fpxUrVuhv1+KcXb9+PeDnPD726NGjFB8fT4MGDaLFixfTq1evbIs3mHO5efPmn9Xo0qVLbYt3oCosLKTZs2frb2wdPXo0rVq1iiorKwOWaW9vp9zcXBoxYgQNGTKE1q5dS2/fvrUt5mDO5aJFi35Wl9u3b7ctZmObgStXrtD+/fv1JR5PnjyhtLQ0WrJkCTU2NloToUGmTp1K9fX1/tuDBw/sDmnAa2tr0zXHDWlPTp48SWfOnKHz58/Tw4cPafDgwbo+eWcMfcsl4zf/7jVaVFT0VWMMBmVlZfqNvqKigm7fvk2dnZ2UlZWl8+uzb98+unnzJhUXF+vl+eve16xZY2vcwZpLlpOTE1CX/LqHPlJC6enpKjc31//Y4/GohIQEVVhYKN20UQoKClRaWprdYQQ1LueSkhL/Y6/Xq+Li4tSpU6f8/9fU1KTCw8NVUVGRTVEGZy5Zdna2WrlypW0xBavGxkadz7KyMn8NhoWFqeLiYv8yL1680MuUl5fbGGnw5ZItXLhQ7dmzx9a4fgtERwY6Ojro8ePH+tBr93kM+HF5eblk00biw9d8iHb8+PG0adMmqq6utjukoPbmzRtqaGgIqE/+jm4+lYX67J979+7pw7WTJ0+mHTt20Pv37+0OacBrbm7W9zExMfqe95n8Cbd7XfIpweTkZNRlH3Ppc+nSJRo5ciRNmzaNDh8+TG6326YIg1evJir6Je/evSOPx0OxsbEB/8+PX758KY3NKPwGdfHiRb2T5cNcx48fpwULFtDz58/1+TLoO24EWE/16fsZ9B6fIuBD2ePGjaPXr1/TkSNHaNmyZfoNLCQkxO7wBiSe8XXv3r00f/58/UbFuPZcLhcNGzYsYFnUZd9zyTZu3EgpKSn6g9SzZ88oLy9Pjyu4du2arfEa1QyAdXin6jNjxgzdHHCBX716lbZu3WprbABs/fr1/n9Pnz5d1+mECRP00YLMzExbYxuo+Hw3N/QY//Plcrlt27aAuuTBwlyP3LByfULviE4T8GEZ/kTw01Gw/DguLk6yaePxp4ZJkyZRVVWV3aEELV8Noj6/DD6dxfsA1GjPdu3aRbdu3aLS0tKAKeC59vgUa1NTU8DyqMu+57In/EGKoS6/YjPAh7pmzpxJd+7cCTiUw4/nzZsn2bTxPnz4oDtb7nKhf/hwNu9cu9dnS0uLvqoA9SlXW1urxwygRgPx+Et+8yopKaG7d+/qOuyO95lhYWEBdcmHtXmMEOqyb7nsydOnT/U96vIrnybgywqzs7Np1qxZlJ6eTqdPn9aXfWzZskW6aaMcOHBAX+PNpwb4MiO+VJOPumzYsMHu0AZ809T9EwAPGuSdAQ8w4gFZfI7xxIkTNHHiRL0jyc/P1+cW+Xpl6H0u+cbjWPh6eG6wuFE9dOgQpaam6ks1IfBw9uXLl+nGjRt6vI9vHAAPXuXvuuB7PvXH+07Oa3R0NO3evVs3AnPnzrU7/KDKJdch/3z58uX6Oxt4zABftpmRkaFPY0EfWHFJwtmzZ1VycrJyuVz6UsOKigorNmuUdevWqfj4eJ3DMWPG6MdVVVV2hzXglZaW6kuNfnrjy+B8lxfm5+er2NhYfUlhZmamqqystDvsoMul2+1WWVlZatSoUfqyuJSUFJWTk6MaGhrsDnvA6SmHfLtw4YJ/mY8fP6qdO3eq4cOHq8jISLV69WpVX19va9zBmMvq6mqVkZGhYmJi9Os7NTVVHTx4UDU3N9sdetDBFMYAAACGw9wEAAAAhkMzAAAAYDg0AwAAAIZDMwAAAGA4NAMAAACGQzMAAABgODQDAAAAhkMzAAAAYDg0AwAAAIZDMwAAAGA4NAMAAACGQzMAAABAZvsv0tAyDV1A7+EAAAAASUVORK5CYII="
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "execution_count": 24
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
